def main():
"""Entry point"""
from niworkflows.nipype import config as ncfg, logging as nlog
from niworkflows.nipype.pipeline.engine import Workflow
from .. import logging
from ..utils.bids import collect_bids_data
from ..workflows.core import build_workflow
from ..utils.misc import check_folder
# Run parser
opts = get_parser().parse_args()
# Retrieve logging level
log_level = int(max(3 - opts.verbose_count, 0) * 10)
if opts.verbose_count > 1:
log_level = int(max(25 - 5 * opts.verbose_count, 1))
print(log_level)
logging.getLogger().setLevel(log_level)
log = logging.getLogger('mriqc.cli')
# Build settings dict
bids_dir = op.abspath(opts.bids_dir)
# Number of processes
n_procs = opts.n_procs
settings = {
'bids_dir': bids_dir,
'write_graph': opts.write_graph,
'testing': opts.testing,
'hmc_afni': opts.hmc_afni,
'hmc_fsl': opts.hmc_fsl,
'fft_spikes_detector': opts.fft_spikes_detector,
'n_procs': n_procs,
'ants_nthreads': opts.ants_nthreads,
'output_dir': op.abspath(opts.output_dir),
'work_dir': op.abspath(opts.work_dir),
'verbose_reports': opts.verbose_reports or opts.testing,
'float32': opts.float32,
'ica': opts.ica,
'no_sub': opts.no_sub,
'email': opts.email,
'fd_thres': opts.fd_thres,
'webapi_url': opts.webapi_url,
'webapi_port': opts.webapi_port,
'upload_strict': opts.upload_strict,
}
if opts.hmc_afni:
settings['deoblique'] = opts.deoblique
settings['despike'] = opts.despike
settings['correct_slice_timing'] = opts.correct_slice_timing
if opts.start_idx:
settings['start_idx'] = opts.start_idx
if opts.stop_idx:
settings['stop_idx'] = opts.stop_idx
if opts.ants_settings:
settings['ants_settings'] = opts.ants_settings
log_dir = op.join(settings['output_dir'], 'logs')
analysis_levels = opts.analysis_level
if opts.participant_label is None:
analysis_levels.append('group')
analysis_levels = list(set(analysis_levels))
if len(analysis_levels) > 2:
raise RuntimeError('Error parsing analysis levels, got "%s"' %
', '.join(analysis_levels))
settings['report_dir'] = opts.report_dir
if not settings['report_dir']:
settings['report_dir'] = op.join(settings['output_dir'], 'reports')
check_folder(settings['output_dir'])
if 'participant' in analysis_levels:
check_folder(settings['work_dir'])
check_folder(log_dir)
check_folder(settings['report_dir'])
# Set nipype config
ncfg.update_config({
'logging': {
'log_directory': log_dir,
'log_to_file': True
},
'execution': {
'crashdump_dir': log_dir,
'crashfile_format': 'txt'
},
})
# Set nipype logging level
nlog.getLogger('workflow').setLevel(log_level)
nlog.getLogger('interface').setLevel(log_level)
nlog.getLogger('filemanip').setLevel(log_level)
callback_log_path = None
plugin_settings = {'plugin': 'Linear'}
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as pfile:
plugin_settings = loadyml(pfile)
else:
# Setup multiprocessing
if settings['n_procs'] == 0:
settings['n_procs'] = cpu_count()
if settings['ants_nthreads'] == 0:
if settings['n_procs'] > 1:
# always leave one extra thread for non ANTs work,
# don't use more than 8 threads - the speed up is minimal
settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8)
else:
settings['ants_nthreads'] = 1
if settings['n_procs'] > 1:
plugin_settings['plugin'] = 'MultiProc'
plugin_settings['plugin_args'] = {'n_procs': settings['n_procs']}
if opts.mem_gb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb
# Process data types
modalities = opts.modalities
dataset = collect_bids_data(
settings['bids_dir'],
modalities=modalities,
participant_label=opts.participant_label,
session=opts.session_id,
run=opts.run_id,
task=opts.task_id,
)
# Set up participant level
if 'participant' in analysis_levels:
log.info('Participant level started...')
log.info(
'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s',
__version__, ', '.join(analysis_levels), opts.participant_label,
settings)
workflow = Workflow(name='workflow_enumerator')
workflow.base_dir = settings['work_dir']
wf_list = []
for mod in modalities:
if not dataset[mod]:
log.warning('No %s scans were found in %s', mod,
settings['bids_dir'])
continue
wf_list.append(build_workflow(dataset[mod], mod,
settings=settings))
if wf_list:
workflow.add_nodes(wf_list)
if not opts.dry_run:
if plugin_settings['plugin'] == 'MultiProc' and opts.profile:
import logging
from niworkflows.nipype.pipeline.plugins.callback_log import log_nodes_cb
plugin_settings['plugin_args'][
'status_callback'] = log_nodes_cb
callback_log_path = op.join(log_dir, 'run_stats.log')
logger = logging.getLogger('callback')
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(callback_log_path)
logger.addHandler(handler)
# Warn about submitting measures BEFORE
if not settings['no_sub']:
log.warning('Anonymized quality metrics will be submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
# run MRIQC
workflow.run(**plugin_settings)
# Warn about submitting measures AFTER
if not settings['no_sub']:
log.warning(
'Anonymized quality metrics have beeen submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
if callback_log_path is not None:
from niworkflows.nipype.utils.draw_gantt_chart import generate_gantt_chart
generate_gantt_chart(callback_log_path,
cores=settings['n_procs'])
else:
msg = """\
Error reading BIDS directory ({}), or the dataset is not \
BIDS-compliant."""
if opts.participant_label is not None:
msg = """\
None of the supplied labels (--participant_label) matched with the \
participants found in the BIDS directory ({})."""
raise RuntimeError(msg.format(settings['bids_dir']))
log.info('Participant level finished successfully.')
# Set up group level
if 'group' in analysis_levels:
from ..reports import group_html
from ..utils.misc import generate_csv # , generate_pred
log.info('Group level started...')
log.info(
'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s',
__version__, ', '.join(analysis_levels), opts.participant_label,
settings)
reports_dir = check_folder(op.join(settings['output_dir'], 'reports'))
derivatives_dir = op.join(settings['output_dir'], 'derivatives')
n_group_reports = 0
for mod in modalities:
dataframe, out_csv = generate_csv(derivatives_dir,
settings['output_dir'], mod)
# If there are no iqm.json files, nothing to do.
if dataframe is None:
log.warning(
'No IQM-JSON files were found for the %s data type in %s. The group-level '
'report was not generated.', mod, derivatives_dir)
continue
log.info('Summary CSV table for the %s data generated (%s)', mod,
out_csv)
# out_pred = generate_pred(derivatives_dir, settings['output_dir'], mod)
# if out_pred is not None:
# log.info('Predicted QA CSV table for the %s data generated (%s)',
# mod, out_pred)
out_html = op.join(reports_dir, mod + '_group.html')
group_html(out_csv,
mod,
csv_failed=op.join(settings['output_dir'],
'failed_' + mod + '.csv'),
out_file=out_html)
log.info('Group-%s report generated (%s)', mod, out_html)
n_group_reports += 1
if n_group_reports == 0:
raise Exception(
"No data found. No group level reports were generated.")
log.info('Group level finished successfully.')
def build_workflow(opts, retval):
"""
Create the Nipype Workflow that supports the whole execution
graph, given the inputs.
All the checks and the construction of the workflow are done
inside this function that has pickleable inputs and output
dictionary (``retval``) to allow isolation using a
``multiprocessing.Process`` that allows fmriprep to enforce
a hard-limited memory-scope.
"""
from niworkflows.nipype import logging, config as ncfg
from ..info import __version__
from ..workflows.base import init_fmriprep_wf
from ..utils.bids import collect_participants
from ..viz.reports import generate_reports
logger = logging.getLogger('workflow')
INIT_MSG = """
Running fMRIPREP version {version}:
* BIDS dataset path: {bids_dir}.
* Participant list: {subject_list}.
* Run identifier: {uuid}.
""".format
# Validity of some inputs
# ERROR check if use_aroma was specified, but the correct template was not
if opts.use_aroma and (opts.template != 'MNI152NLin2009cAsym'
or 'template' not in opts.output_space):
raise RuntimeError(
'ERROR: --use-aroma requires functional images to be resampled to '
'MNI152NLin2009cAsym.\n'
'\t--template must be set to "MNI152NLin2009cAsym" (was: "{}")\n'
'\t--output-space list must include "template" (was: "{}")'.format(
opts.template, ' '.join(opts.output_space)))
# Check output_space
if 'template' not in opts.output_space and (opts.use_syn_sdc
or opts.force_syn):
msg = ('SyN SDC correction requires T1 to MNI registration, but '
'"template" is not specified in "--output-space" arguments')
if opts.force_syn:
raise RuntimeError(msg)
logger.warning(msg)
# Set up some instrumental utilities
run_uuid = '%s_%s' % (strftime('%Y%m%d-%H%M%S'), uuid.uuid4())
# First check that bids_dir looks like a BIDS folder
bids_dir = op.abspath(opts.bids_dir)
subject_list = collect_participants(
bids_dir, participant_label=opts.participant_label)
# Setting up MultiProc
nthreads = opts.nthreads
if nthreads < 1:
nthreads = cpu_count()
plugin_settings = {
'plugin': 'MultiProc',
'plugin_args': {
'n_procs': nthreads,
'raise_insufficient': False,
'maxtasksperchild': 1,
}
}
if opts.mem_mb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_mb / 1024
# Overload plugin_settings if --use-plugin
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as f:
plugin_settings = loadyml(f)
omp_nthreads = opts.omp_nthreads
if omp_nthreads == 0:
omp_nthreads = min(nthreads - 1 if nthreads > 1 else cpu_count(), 8)
if 1 < nthreads < omp_nthreads:
logger.warning(
'Per-process threads (--omp-nthreads=%d) exceed total '
'threads (--nthreads/--n_cpus=%d)', omp_nthreads, nthreads)
# Set up directories
output_dir = op.abspath(opts.output_dir)
log_dir = op.join(output_dir, 'fmriprep', 'logs')
work_dir = op.abspath(opts.work_dir or 'work') # Set work/ as default
# Check and create output and working directories
os.makedirs(output_dir, exist_ok=True)
os.makedirs(log_dir, exist_ok=True)
os.makedirs(work_dir, exist_ok=True)
# Nipype config (logs and execution)
ncfg.update_config({
'logging': {
'log_directory': log_dir,
'log_to_file': True
},
'execution': {
'crashdump_dir':
log_dir,
'crashfile_format':
'txt',
'get_linked_libs':
False,
'stop_on_first_crash':
opts.stop_on_first_crash or opts.work_dir is None,
},
'monitoring': {
'enabled': opts.resource_monitor,
'sample_frequency': '0.5',
'summary_append': True,
}
})
if opts.resource_monitor:
ncfg.enable_resource_monitor()
retval['return_code'] = 0
retval['plugin_settings'] = plugin_settings
retval['output_dir'] = output_dir
retval['work_dir'] = work_dir
retval['subject_list'] = subject_list
retval['run_uuid'] = run_uuid
retval['workflow'] = None
# Called with reports only
if opts.reports_only:
logger.log(25, 'Running --reports-only on participants %s',
', '.join(subject_list))
if opts.run_uuid is not None:
run_uuid = opts.run_uuid
retval['return_code'] = generate_reports(subject_list, output_dir,
work_dir, run_uuid)
return retval
# Build main workflow
logger.log(
25,
INIT_MSG(version=__version__,
bids_dir=bids_dir,
subject_list=subject_list,
uuid=run_uuid))
retval['workflow'] = init_fmriprep_wf(
subject_list=subject_list,
task_id=opts.task_id,
run_uuid=run_uuid,
ignore=opts.ignore,
debug=opts.debug,
low_mem=opts.low_mem,
anat_only=opts.anat_only,
longitudinal=opts.longitudinal,
t2s_coreg=opts.t2s_coreg,
omp_nthreads=omp_nthreads,
skull_strip_template=opts.skull_strip_template,
work_dir=work_dir,
output_dir=output_dir,
bids_dir=bids_dir,
freesurfer=opts.run_reconall,
output_spaces=opts.output_space,
template=opts.template,
medial_surface_nan=opts.medial_surface_nan,
output_grid_ref=opts.output_grid_reference,
hires=opts.hires,
use_bbr=opts.use_bbr,
bold2t1w_dof=opts.bold2t1w_dof,
fmap_bspline=opts.fmap_bspline,
fmap_demean=opts.fmap_no_demean,
use_syn=opts.use_syn_sdc,
force_syn=opts.force_syn,
use_aroma=opts.use_aroma,
ignore_aroma_err=opts.ignore_aroma_denoising_errors,
)
retval['return_code'] = 0
return retval
def create_workflow(opts):
"""Build workflow"""
from niworkflows.nipype import config as ncfg
from ..viz.reports import run_reports
from ..workflows.base import init_fmriprep_wf
from ..utils.bids import collect_participants
# Set up some instrumental utilities
errno = 0
run_uuid = strftime('%Y%m%d-%H%M%S_') + str(uuid.uuid4())
# First check that bids_dir looks like a BIDS folder
bids_dir = op.abspath(opts.bids_dir)
subject_list = collect_participants(
bids_dir, participant_label=opts.participant_label)
# Nipype plugin configuration
plugin_settings = {'plugin': 'Linear'}
nthreads = opts.nthreads
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as f:
plugin_settings = loadyml(f)
else:
# Setup multiprocessing
nthreads = opts.nthreads
if nthreads == 0:
nthreads = cpu_count()
if nthreads > 1:
plugin_settings['plugin'] = 'MultiProc'
plugin_settings['plugin_args'] = {'n_procs': nthreads}
if opts.mem_mb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_mb / 1024
omp_nthreads = opts.omp_nthreads
if omp_nthreads == 0:
omp_nthreads = min(nthreads - 1 if nthreads > 1 else cpu_count(), 8)
if 1 < nthreads < omp_nthreads:
raise RuntimeError(
'Per-process threads (--omp-nthreads={:d}) cannot exceed total '
'threads (--nthreads/--n_cpus={:d})'.format(omp_nthreads, nthreads))
# Set up directories
output_dir = op.abspath(opts.output_dir)
log_dir = op.join(output_dir, 'fmriprep', 'logs')
work_dir = op.abspath(opts.work_dir)
# Check and create output and working directories
os.makedirs(output_dir, exist_ok=True)
os.makedirs(log_dir, exist_ok=True)
os.makedirs(work_dir, exist_ok=True)
# Nipype config (logs and execution)
ncfg.update_config({
'logging': {'log_directory': log_dir, 'log_to_file': True},
'execution': {'crashdump_dir': log_dir, 'crashfile_format': 'txt'},
})
# Called with reports only
if opts.reports_only:
logger.log(25, 'Running --reports-only on participants %s', ', '.join(subject_list))
report_errors = [
run_reports(op.join(work_dir, 'reportlets'), output_dir, subject_label,
run_uuid=run_uuid)
for subject_label in subject_list]
sys.exit(int(sum(report_errors) > 0))
# Build main workflow
logger.log(25, INIT_MSG(
version=__version__,
subject_list=subject_list,
uuid=run_uuid)
)
fmriprep_wf = init_fmriprep_wf(
subject_list=subject_list,
task_id=opts.task_id,
run_uuid=run_uuid,
ignore=opts.ignore,
debug=opts.debug,
low_mem=opts.low_mem,
anat_only=opts.anat_only,
longitudinal=opts.longitudinal,
omp_nthreads=omp_nthreads,
skull_strip_ants=True,
work_dir=work_dir,
output_dir=output_dir,
bids_dir=bids_dir,
freesurfer=opts.freesurfer,
output_spaces=opts.output_space,
template=opts.template,
medial_surface_nan=opts.medial_surface_nan,
output_grid_ref=opts.output_grid_reference,
hires=opts.hires,
bold2t1w_dof=opts.bold2t1w_dof,
fmap_bspline=opts.fmap_bspline,
fmap_demean=opts.fmap_no_demean,
use_syn=opts.use_syn_sdc,
force_syn=opts.force_syn,
use_aroma=opts.use_aroma,
ignore_aroma_err=opts.ignore_aroma_denoising_errors,
)
if opts.write_graph:
fmriprep_wf.write_graph(graph2use="colored", format='svg', simple_form=True)
try:
fmriprep_wf.run(**plugin_settings)
except RuntimeError as e:
if "Workflow did not execute cleanly" in str(e):
errno = 1
else:
raise(e)
# Generate reports phase
report_errors = [run_reports(
op.join(work_dir, 'reportlets'), output_dir, subject_label, run_uuid=run_uuid)
for subject_label in subject_list]
if sum(report_errors):
logger.warning('Errors occurred while generating reports for participants: %s.',
', '.join(['%s (%d)' % (subid, err)
for subid, err in zip(subject_list, report_errors)]))
errno += sum(report_errors)
sys.exit(int(errno > 0))
def main():
"""Entry point"""
from niworkflows.nipype import config as ncfg, logging as nlog
from niworkflows.nipype.pipeline.engine import Workflow
from .. import logging
from ..utils.bids import collect_bids_data
from ..workflows.core import build_workflow
from ..utils.misc import check_folder
# Run parser
opts = get_parser().parse_args()
# Retrieve logging level
log_level = int(max(3 - opts.verbose_count, 0) * 10)
if opts.verbose_count > 1:
log_level = int(max(25 - 5 * opts.verbose_count, 1))
logging.getLogger().setLevel(log_level)
log = logging.getLogger('mriqc.cli')
# Build settings dict
bids_dir = op.abspath(opts.bids_dir)
# Number of processes
n_procs = opts.n_procs
settings = {
'bids_dir': bids_dir,
'write_graph': opts.write_graph,
'testing': opts.testing,
'hmc_afni': opts.hmc_afni,
'hmc_fsl': opts.hmc_fsl,
'fft_spikes_detector': opts.fft_spikes_detector,
'n_procs': n_procs,
'ants_nthreads': opts.ants_nthreads,
'ants_float': opts.ants_float,
'output_dir': op.abspath(opts.output_dir),
'work_dir': op.abspath(opts.work_dir),
'verbose_reports': opts.verbose_reports or opts.testing,
'float32': opts.float32,
'ica': opts.ica,
'no_sub': opts.no_sub,
'email': opts.email,
'fd_thres': opts.fd_thres,
'webapi_url': opts.webapi_url,
'webapi_port': opts.webapi_port,
'upload_strict': opts.upload_strict,
}
if opts.hmc_afni:
settings['deoblique'] = opts.deoblique
settings['despike'] = opts.despike
settings['correct_slice_timing'] = opts.correct_slice_timing
if opts.start_idx:
settings['start_idx'] = opts.start_idx
if opts. stop_idx:
settings['stop_idx'] = opts.stop_idx
if opts.ants_settings:
settings['ants_settings'] = opts.ants_settings
log_dir = op.join(settings['output_dir'], 'logs')
analysis_levels = opts.analysis_level
if opts.participant_label is None:
analysis_levels.append('group')
analysis_levels = list(set(analysis_levels))
if len(analysis_levels) > 2:
raise RuntimeError('Error parsing analysis levels, got "%s"' % ', '.join(analysis_levels))
settings['report_dir'] = opts.report_dir
if not settings['report_dir']:
settings['report_dir'] = op.join(settings['output_dir'], 'reports')
check_folder(settings['output_dir'])
if 'participant' in analysis_levels:
check_folder(settings['work_dir'])
# check_folder(log_dir)
# check_folder(settings['report_dir'])
# Set nipype config
ncfg.update_config({
# 'logging': {'log_directory': log_dir, 'log_to_file': True},
'execution': {#'crashdump_dir': log_dir,
'crashfile_format': 'txt',
'resource_monitor': opts.profile},
})
# Set nipype logging level
nlog.getLogger('workflow').setLevel(log_level)
nlog.getLogger('interface').setLevel(log_level)
nlog.getLogger('utils').setLevel(log_level)
plugin_settings = {'plugin': 'Linear'}
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as pfile:
plugin_settings = loadyml(pfile)
else:
# Setup multiprocessing
if settings['n_procs'] == 0:
settings['n_procs'] = cpu_count()
if settings['ants_nthreads'] == 0:
if settings['n_procs'] > 1:
# always leave one extra thread for non ANTs work,
# don't use more than 8 threads - the speed up is minimal
settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8)
else:
settings['ants_nthreads'] = 1
if settings['n_procs'] > 1:
plugin_settings['plugin'] = 'MultiProc'
plugin_settings['plugin_args'] = {'n_procs': settings['n_procs']}
if opts.mem_gb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb
# Process data types
modalities = opts.modalities
dataset = collect_bids_data(
settings['bids_dir'],
modalities=modalities,
participant_label=opts.participant_label,
session=opts.session_id,
run=opts.run_id,
task=opts.task_id,
)
# Set up participant level
if 'participant' in analysis_levels:
log.info('Participant level started...')
log.info(
'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s',
__version__, ', '.join(analysis_levels), opts.participant_label, settings)
workflow = Workflow(name='workflow_enumerator')
workflow.base_dir = settings['work_dir']
wf_list = []
for mod in modalities:
if not dataset[mod]:
log.warning('No %s scans were found in %s', mod, settings['bids_dir'])
continue
wf_list.append(build_workflow(dataset[mod], mod, settings=settings))
if wf_list:
workflow.add_nodes(wf_list)
if not opts.dry_run:
# Warn about submitting measures BEFORE
if not settings['no_sub']:
log.warning(
'Anonymized quality metrics will be submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
# run MRIQC
workflow.run(**plugin_settings)
# Warn about submitting measures AFTER
if not settings['no_sub']:
log.warning(
'Anonymized quality metrics have beeen submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
else:
msg = 'Error reading BIDS directory ({}), or the dataset is not ' \
'BIDS-compliant.'
if opts.participant_label or opts.session_id or opts.run_id or opts.task_id:
msg = 'The combination of supplied labels'
if opts.participant_label is not None:
msg += ' (--participant_label {})'.format(" ".join(opts.participant_label))
if opts.session_id is not None:
msg += ' (--session-id {})'.format(" ".join(opts.session_id))
if opts.run_id is not None:
msg += ' (--run-id {})'.format(" ".join(opts.run_id))
if opts.task_id is not None:
msg += ' (--task-id {})'.format(" ".join(opts.task_id))
msg += ' did not result in matches within the BIDS directory ({}).'
raise RuntimeError(msg.format(settings['bids_dir']))
log.info('Participant level finished successfully.')
for mod in modalities:
dataframe, order, jsonfiles = generate_csv(settings['output_dir'], mod)
# If there are no iqm.json files, nothing to do.
if dataframe is None:
log.warning(
'No IQM-JSON files were found for the %s data type in %s. The group-level '
'report was not generated.', mod, settings['output_dir'])
continue
if 'jsonfile' in order: order.remove('jsonfile')
base_name = 'mclf_run-20170724-191452_mod-rfc_ver-0.9.7-rc8_class-2_cv-loso'
load_classifier = pkgrf(
'mriqc',
'data/mclf_run-20170724-191452_mod-rfc_ver-0.9.7-rc8_class-2_cv-'
'loso_data-all_estimator.pklz')
cvhelper = CVHelper(load_clf=load_classifier, n_jobs=-1,
rate_label=['rater_1'], basename=base_name)
prediction = cvhelper.predict(dataframe[order])
dataframe['y_prob'] = prediction[0][:,1]
dataframe['y_pred'] = prediction[1]
dataframe.index = jsonfiles
for jsonfile in dataframe.index.values:
with open(jsonfile, 'r+') as json_file:
json_dict = json.load(json_file, object_pairs_hook=OrderedDict)
json_dict['y_prob'] = float(dataframe.get_value(jsonfile, 'y_prob'))
json_dict['y_pred'] = float(dataframe.get_value(jsonfile, 'y_pred'))
json_file.seek(0)
json.dump(json_dict, json_file, separators=(',', ':'), sort_keys=True, indent=4)
json_file.truncate()
def main():
"""Entry point"""
from niworkflows.nipype import config as ncfg, logging as nlog
from niworkflows.nipype.pipeline.engine import Workflow
from .. import logging
from ..utils.bids import collect_bids_data
from ..workflows.core import build_workflow
from ..utils.misc import check_folder
# Run parser
opts = get_parser().parse_args()
# Retrieve logging level
log_level = int(max(3 - opts.verbose_count, 0) * 10)
if opts.verbose_count > 1:
log_level = int(max(25 - 5 * opts.verbose_count, 1))
print(log_level)
logging.getLogger().setLevel(log_level)
log = logging.getLogger('mriqc.cli')
# Build settings dict
bids_dir = op.abspath(opts.bids_dir)
# Number of processes
n_procs = opts.n_procs
settings = {
'bids_dir': bids_dir,
'write_graph': opts.write_graph,
'testing': opts.testing,
'hmc_afni': opts.hmc_afni,
'hmc_fsl': opts.hmc_fsl,
'fft_spikes_detector': opts.fft_spikes_detector,
'n_procs': n_procs,
'ants_nthreads': opts.ants_nthreads,
'output_dir': op.abspath(opts.output_dir),
'work_dir': op.abspath(opts.work_dir),
'verbose_reports': opts.verbose_reports or opts.testing,
'float32': opts.float32,
'ica': opts.ica,
'no_sub': opts.no_sub,
'email': opts.email,
'fd_thres': opts.fd_thres,
'webapi_url' : opts.webapi_url,
'webapi_port' : opts.webapi_port,
'upload_strict' : opts.upload_strict,
}
if opts.hmc_afni:
settings['deoblique'] = opts.deoblique
settings['despike'] = opts.despike
settings['correct_slice_timing'] = opts.correct_slice_timing
if opts.start_idx:
settings['start_idx'] = opts.start_idx
if opts. stop_idx:
settings['stop_idx'] = opts.stop_idx
if opts.ants_settings:
settings['ants_settings'] = opts.ants_settings
log_dir = op.join(settings['output_dir'], 'logs')
analysis_levels = opts.analysis_level
if opts.participant_label is None:
analysis_levels.append('group')
analysis_levels = list(set(analysis_levels))
if len(analysis_levels) > 2:
raise RuntimeError('Error parsing analysis levels, got "%s"' % ', '.join(analysis_levels))
settings['report_dir'] = opts.report_dir
if not settings['report_dir']:
settings['report_dir'] = op.join(settings['output_dir'], 'reports')
check_folder(settings['output_dir'])
if 'participant' in analysis_levels:
check_folder(settings['work_dir'])
check_folder(log_dir)
check_folder(settings['report_dir'])
# Set nipype config
ncfg.update_config({
'logging': {'log_directory': log_dir, 'log_to_file': True},
'execution': {'crashdump_dir': log_dir, 'crashfile_format': 'txt'},
})
# Set nipype logging level
nlog.getLogger('workflow').setLevel(log_level)
nlog.getLogger('interface').setLevel(log_level)
nlog.getLogger('filemanip').setLevel(log_level)
callback_log_path = None
plugin_settings = {'plugin': 'Linear'}
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as pfile:
plugin_settings = loadyml(pfile)
else:
# Setup multiprocessing
if settings['n_procs'] == 0:
settings['n_procs'] = cpu_count()
if settings['ants_nthreads'] == 0:
if settings['n_procs'] > 1:
# always leave one extra thread for non ANTs work,
# don't use more than 8 threads - the speed up is minimal
settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8)
else:
settings['ants_nthreads'] = 1
if settings['n_procs'] > 1:
plugin_settings['plugin'] = 'MultiProc'
plugin_settings['plugin_args'] = {'n_procs': settings['n_procs']}
if opts.mem_gb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb
# Process data types
modalities = opts.modalities
dataset = collect_bids_data(
settings['bids_dir'],
modalities=modalities,
participant_label=opts.participant_label,
session=opts.session_id,
run=opts.run_id,
task=opts.task_id,
)
# Set up participant level
if 'participant' in analysis_levels:
log.info('Participant level started...')
log.info(
'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s',
__version__, ', '.join(analysis_levels), opts.participant_label, settings)
workflow = Workflow(name='workflow_enumerator')
workflow.base_dir = settings['work_dir']
wf_list = []
for mod in modalities:
if not dataset[mod]:
log.warning('No %s scans were found in %s', mod, settings['bids_dir'])
continue
wf_list.append(build_workflow(dataset[mod], mod, settings=settings))
if wf_list:
workflow.add_nodes(wf_list)
if not opts.dry_run:
if plugin_settings['plugin'] == 'MultiProc' and opts.profile:
import logging
from niworkflows.nipype.pipeline.plugins.callback_log import log_nodes_cb
plugin_settings['plugin_args']['status_callback'] = log_nodes_cb
callback_log_path = op.join(log_dir, 'run_stats.log')
logger = logging.getLogger('callback')
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(callback_log_path)
logger.addHandler(handler)
# Warn about submitting measures BEFORE
if not settings['no_sub']:
log.warning(
'Anonymized quality metrics will be submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
# run MRIQC
workflow.run(**plugin_settings)
# Warn about submitting measures AFTER
if not settings['no_sub']:
log.warning(
'Anonymized quality metrics have beeen submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
if callback_log_path is not None:
from niworkflows.nipype.utils.draw_gantt_chart import generate_gantt_chart
generate_gantt_chart(callback_log_path, cores=settings['n_procs'])
else:
msg = """\
Error reading BIDS directory ({}), or the dataset is not \
BIDS-compliant."""
if opts.participant_label is not None:
msg = """\
None of the supplied labels (--participant_label) matched with the \
participants found in the BIDS directory ({})."""
raise RuntimeError(msg.format(settings['bids_dir']))
log.info('Participant level finished successfully.')
# Set up group level
if 'group' in analysis_levels:
from ..reports import group_html
from ..utils.misc import generate_csv # , generate_pred
log.info('Group level started...')
log.info(
'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s',
__version__, ', '.join(analysis_levels), opts.participant_label, settings)
reports_dir = check_folder(op.join(settings['output_dir'], 'reports'))
derivatives_dir = op.join(settings['output_dir'], 'derivatives')
n_group_reports = 0
for mod in modalities:
dataframe, out_csv = generate_csv(derivatives_dir,
settings['output_dir'], mod)
# If there are no iqm.json files, nothing to do.
if dataframe is None:
log.warning(
'No IQM-JSON files were found for the %s data type in %s. The group-level '
'report was not generated.', mod, derivatives_dir)
continue
log.info('Summary CSV table for the %s data generated (%s)', mod, out_csv)
# out_pred = generate_pred(derivatives_dir, settings['output_dir'], mod)
# if out_pred is not None:
# log.info('Predicted QA CSV table for the %s data generated (%s)',
# mod, out_pred)
out_html = op.join(reports_dir, mod + '_group.html')
group_html(out_csv, mod,
csv_failed=op.join(settings['output_dir'], 'failed_' + mod + '.csv'),
out_file=out_html)
log.info('Group-%s report generated (%s)', mod, out_html)
n_group_reports += 1
if n_group_reports == 0:
raise Exception("No data found. No group level reports were generated.")
log.info('Group level finished successfully.')
def main():
# Get commandline arguments
opts = get_parser().parse_args()
# get process options to name output directory
if opts.smooth:
smooth_name = '_{}mm'.format(opts.smooth)
else:
smooth_name = ''
if opts.low_pass:
lp_name = '_{}Hz'.format(opts.low_pass)
else:
lp_name = ''
if opts.regfilt:
regfilt_name = '_regfilt'
else:
regfilt_name = ''
# Set up main directories
output_dir = os.path.abspath(opts.output_dir)
result_dir = os.path.join(
output_dir, 'NuisanceRegression{}{}{}'.format(smooth_name, lp_name,
regfilt_name))
log_dir = os.path.join(result_dir, 'logs')
work_dir = os.path.abspath(opts.work_dir)
# make directories
# Check and create output and working directories
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
if not os.path.exists(work_dir):
os.makedirs(work_dir)
# Set up some instrumental utilities
run_uuid = strftime('%Y%m%d-%H%M%S_') + str(uuid.uuid4())
# Nipype config (logs and execution)
ncfg.update_config({
'logging': {
'log_directory': log_dir,
'log_to_file': True
},
'execution': {
'crashdump_dir': log_dir,
'crashfile_format': 'txt',
'parameterize_dirs': False,
},
})
# Nipype plugin configuration
plugin_settings = {'plugin': 'Linear'}
# only for a subset of subjects
if opts.participant_label:
subject_list = opts.participant_label
# for all subjects
else:
subject_dirs = [
directory
for directory in glob(os.path.join(opts.deriv_pipe_dir, "sub-*"))
if os.path.isdir(os.path.join(opts.deriv_pipe_dir, directory))
]
subject_list = [
subject_dir.split("-")[-1] for subject_dir in subject_dirs
]
nuisance_regression_wf = init_nuisance_regression_wf(
confound_names=opts.confounds,
deriv_pipe_dir=opts.deriv_pipe_dir,
exclude_variant=opts.exclude_variant,
low_pass=opts.low_pass,
regfilt=opts.regfilt,
res=opts.res,
result_dir=result_dir,
run_id=opts.run,
run_uuid=run_uuid,
ses_id=opts.ses,
smooth=opts.smooth,
space=opts.space,
subject_list=subject_list,
task_id=opts.task_id,
variant=opts.variant,
work_dir=work_dir,
)
if opts.graph:
nuisance_regression_wf.write_graph(graph2use='colored',
dotfilename=os.path.join(
work_dir, 'graph_colored.dot'))
try:
nuisance_regression_wf.run(**plugin_settings)
except RuntimeError as e:
print('ERROR!')
raise (e)