def get_final_cost(in_file):
from niworkflows.nipype import logging
with open(in_file, 'r') as fobj:
for line in fobj:
if line.startswith(' >> print U:1'):
costs = next(fobj).split()
return float(costs[0])
logger = logging.getLogger('interface')
logger.error('No cost report found in log file. Please report this '
'issue, with contents of {}'.format(in_file))
def _new_version():
from niworkflows.nipype import logging
iflogger = logging.getLogger('interface')
level = iflogger.getEffectiveLevel()
iflogger.setLevel('ERROR')
v = _old_version()
iflogger.setLevel(level)
if v is None:
iflogger.warn('afni_vcheck executable not found')
return v
def get_final_cost(in_file):
from niworkflows.nipype import logging
with open(in_file, 'r') as fobj:
for line in fobj:
if line.startswith('>> print U:1'):
costs = next(fobj).split()
return float(costs[0])
logger = logging.getLogger('interface')
logger.error('No cost report found in log file. Please report this '
'issue, with contents of {}'.format(in_file))
# BOLD workflows
from .confounds import init_bold_confs_wf, init_carpetplot_wf
from .hmc import init_bold_hmc_wf
from .stc import init_bold_stc_wf
from .t2s import init_bold_t2s_wf
from .registration import init_bold_reg_wf
from .resampling import (
init_bold_surf_wf,
init_bold_mni_trans_wf,
init_bold_preproc_trans_wf,
)
from .util import init_bold_reference_wf
DEFAULT_MEMORY_MIN_GB = 0.01
LOGGER = logging.getLogger('workflow')
def init_func_preproc_wf(bold_file, ignore, freesurfer,
use_bbr, t2s_coreg, bold2t1w_dof, reportlets_dir,
output_spaces, template, output_dir, omp_nthreads,
fmap_bspline, fmap_demean, use_syn, force_syn,
use_aroma, ignore_aroma_err, aroma_melodic_dim,
medial_surface_nan, cifti_output,
debug, low_mem, template_out_grid, layout=None):
"""
This workflow controls the functional preprocessing stages of FMRIPREP.
.. workflow::
:graph2use: orig
:simple_form: yes
from os import path as op
import numpy as np
import nibabel as nb
from nilearn.signal import clean
from builtins import zip
from niworkflows.nipype.interfaces.base import (
traits, TraitedSpec, File, isdefined, BaseInterfaceInputSpec,
SimpleInterface
)
from niworkflows.nipype import logging
from ..utils.misc import _flatten_dict
from ..qc.anatomical import snr, fber, efc, summary_stats
from ..qc.functional import gsr
IFLOGGER = logging.getLogger('interface')
class FunctionalQCInputSpec(BaseInterfaceInputSpec):
in_epi = File(exists=True, mandatory=True, desc='input EPI file')
in_hmc = File(exists=True, mandatory=True, desc='input motion corrected file')
in_tsnr = File(exists=True, mandatory=True, desc='input tSNR volume')
in_mask = File(exists=True, mandatory=True, desc='input mask')
direction = traits.Enum('all', 'x', 'y', '-x', '-y', usedefault=True,
desc='direction for GSR computation')
in_fd = File(exists=True, mandatory=True, desc='motion parameters for FD computation')
fd_thres = traits.Float(0.2, usedefault=True, desc='motion threshold for FD computation')
in_dvars = File(exists=True, mandatory=True, desc='input file containing DVARS')
in_fwhm = traits.List(traits.Float, mandatory=True,
desc='smoothness estimated with AFNI')
from os import path as op
import numpy as np
import nibabel as nb
from nilearn.signal import clean
from builtins import zip
from niworkflows.nipype.interfaces.base import (
traits, TraitedSpec, File, isdefined, BaseInterfaceInputSpec)
from niworkflows.nipype import logging
from niworkflows.interfaces.base import SimpleInterface
from ..utils.misc import _flatten_dict
from ..qc.anatomical import snr, fber, efc, summary_stats
from ..qc.functional import gsr
IFLOGGER = logging.getLogger('interface')
class FunctionalQCInputSpec(BaseInterfaceInputSpec):
in_epi = File(exists=True, mandatory=True, desc='input EPI file')
in_hmc = File(exists=True, mandatory=True, desc='input motion corrected file')
in_tsnr = File(exists=True, mandatory=True, desc='input tSNR volume')
in_mask = File(exists=True, mandatory=True, desc='input mask')
direction = traits.Enum('all', 'x', 'y', '-x', '-y', usedefault=True,
desc='direction for GSR computation')
in_fd = File(exists=True, mandatory=True, desc='motion parameters for FD computation')
fd_thres = traits.Float(0.2, usedefault=True, desc='motion threshold for FD computation')
in_dvars = File(exists=True, mandatory=True, desc='input file containing DVARS')
in_fwhm = traits.List(traits.Float, mandatory=True,
desc='smoothness estimated with AFNI')
def main():
"""Entry point"""
from niworkflows.nipype import config as ncfg, logging as nlog
from niworkflows.nipype.pipeline.engine import Workflow
from .. import logging
from ..utils.bids import collect_bids_data
from ..workflows.core import build_workflow
from ..utils.misc import check_folder
# Run parser
opts = get_parser().parse_args()
# Retrieve logging level
log_level = int(max(3 - opts.verbose_count, 0) * 10)
if opts.verbose_count > 1:
log_level = int(max(25 - 5 * opts.verbose_count, 1))
print(log_level)
logging.getLogger().setLevel(log_level)
log = logging.getLogger('mriqc.cli')
# Build settings dict
bids_dir = op.abspath(opts.bids_dir)
# Number of processes
n_procs = opts.n_procs
settings = {
'bids_dir': bids_dir,
'write_graph': opts.write_graph,
'testing': opts.testing,
'hmc_afni': opts.hmc_afni,
'hmc_fsl': opts.hmc_fsl,
'fft_spikes_detector': opts.fft_spikes_detector,
'n_procs': n_procs,
'ants_nthreads': opts.ants_nthreads,
'output_dir': op.abspath(opts.output_dir),
'work_dir': op.abspath(opts.work_dir),
'verbose_reports': opts.verbose_reports or opts.testing,
'float32': opts.float32,
'ica': opts.ica,
'no_sub': opts.no_sub,
'email': opts.email,
'fd_thres': opts.fd_thres,
'webapi_url': opts.webapi_url,
'webapi_port': opts.webapi_port,
'upload_strict': opts.upload_strict,
}
if opts.hmc_afni:
settings['deoblique'] = opts.deoblique
settings['despike'] = opts.despike
settings['correct_slice_timing'] = opts.correct_slice_timing
if opts.start_idx:
settings['start_idx'] = opts.start_idx
if opts.stop_idx:
settings['stop_idx'] = opts.stop_idx
if opts.ants_settings:
settings['ants_settings'] = opts.ants_settings
log_dir = op.join(settings['output_dir'], 'logs')
analysis_levels = opts.analysis_level
if opts.participant_label is None:
analysis_levels.append('group')
analysis_levels = list(set(analysis_levels))
if len(analysis_levels) > 2:
raise RuntimeError('Error parsing analysis levels, got "%s"' %
', '.join(analysis_levels))
settings['report_dir'] = opts.report_dir
if not settings['report_dir']:
settings['report_dir'] = op.join(settings['output_dir'], 'reports')
check_folder(settings['output_dir'])
if 'participant' in analysis_levels:
check_folder(settings['work_dir'])
check_folder(log_dir)
check_folder(settings['report_dir'])
# Set nipype config
ncfg.update_config({
'logging': {
'log_directory': log_dir,
'log_to_file': True
},
'execution': {
'crashdump_dir': log_dir,
'crashfile_format': 'txt'
},
})
# Set nipype logging level
nlog.getLogger('workflow').setLevel(log_level)
nlog.getLogger('interface').setLevel(log_level)
nlog.getLogger('filemanip').setLevel(log_level)
callback_log_path = None
plugin_settings = {'plugin': 'Linear'}
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as pfile:
plugin_settings = loadyml(pfile)
else:
# Setup multiprocessing
if settings['n_procs'] == 0:
settings['n_procs'] = cpu_count()
if settings['ants_nthreads'] == 0:
if settings['n_procs'] > 1:
# always leave one extra thread for non ANTs work,
# don't use more than 8 threads - the speed up is minimal
settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8)
else:
settings['ants_nthreads'] = 1
if settings['n_procs'] > 1:
plugin_settings['plugin'] = 'MultiProc'
plugin_settings['plugin_args'] = {'n_procs': settings['n_procs']}
if opts.mem_gb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb
# Process data types
modalities = opts.modalities
dataset = collect_bids_data(
settings['bids_dir'],
modalities=modalities,
participant_label=opts.participant_label,
session=opts.session_id,
run=opts.run_id,
task=opts.task_id,
)
# Set up participant level
if 'participant' in analysis_levels:
log.info('Participant level started...')
log.info(
'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s',
__version__, ', '.join(analysis_levels), opts.participant_label,
settings)
workflow = Workflow(name='workflow_enumerator')
workflow.base_dir = settings['work_dir']
wf_list = []
for mod in modalities:
if not dataset[mod]:
log.warning('No %s scans were found in %s', mod,
settings['bids_dir'])
continue
wf_list.append(build_workflow(dataset[mod], mod,
settings=settings))
if wf_list:
workflow.add_nodes(wf_list)
if not opts.dry_run:
if plugin_settings['plugin'] == 'MultiProc' and opts.profile:
import logging
from niworkflows.nipype.pipeline.plugins.callback_log import log_nodes_cb
plugin_settings['plugin_args'][
'status_callback'] = log_nodes_cb
callback_log_path = op.join(log_dir, 'run_stats.log')
logger = logging.getLogger('callback')
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(callback_log_path)
logger.addHandler(handler)
# Warn about submitting measures BEFORE
if not settings['no_sub']:
log.warning('Anonymized quality metrics will be submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
# run MRIQC
workflow.run(**plugin_settings)
# Warn about submitting measures AFTER
if not settings['no_sub']:
log.warning(
'Anonymized quality metrics have beeen submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
if callback_log_path is not None:
from niworkflows.nipype.utils.draw_gantt_chart import generate_gantt_chart
generate_gantt_chart(callback_log_path,
cores=settings['n_procs'])
else:
msg = """\
Error reading BIDS directory ({}), or the dataset is not \
BIDS-compliant."""
if opts.participant_label is not None:
msg = """\
None of the supplied labels (--participant_label) matched with the \
participants found in the BIDS directory ({})."""
raise RuntimeError(msg.format(settings['bids_dir']))
log.info('Participant level finished successfully.')
# Set up group level
if 'group' in analysis_levels:
from ..reports import group_html
from ..utils.misc import generate_csv # , generate_pred
log.info('Group level started...')
log.info(
'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s',
__version__, ', '.join(analysis_levels), opts.participant_label,
settings)
reports_dir = check_folder(op.join(settings['output_dir'], 'reports'))
derivatives_dir = op.join(settings['output_dir'], 'derivatives')
n_group_reports = 0
for mod in modalities:
dataframe, out_csv = generate_csv(derivatives_dir,
settings['output_dir'], mod)
# If there are no iqm.json files, nothing to do.
if dataframe is None:
log.warning(
'No IQM-JSON files were found for the %s data type in %s. The group-level '
'report was not generated.', mod, derivatives_dir)
continue
log.info('Summary CSV table for the %s data generated (%s)', mod,
out_csv)
# out_pred = generate_pred(derivatives_dir, settings['output_dir'], mod)
# if out_pred is not None:
# log.info('Predicted QA CSV table for the %s data generated (%s)',
# mod, out_pred)
out_html = op.join(reports_dir, mod + '_group.html')
group_html(out_csv,
mod,
csv_failed=op.join(settings['output_dir'],
'failed_' + mod + '.csv'),
out_file=out_html)
log.info('Group-%s report generated (%s)', mod, out_html)
n_group_reports += 1
if n_group_reports == 0:
raise Exception(
"No data found. No group level reports were generated.")
log.info('Group level finished successfully.')
def main():
"""Entry point"""
from niworkflows.nipype import logging as nlogging
from multiprocessing import set_start_method, Process, Manager
from ..viz.reports import generate_reports
set_start_method('forkserver')
warnings.showwarning = _warn_redirect
opts = get_parser().parse_args()
# FreeSurfer license
default_license = op.join(os.getenv('FREESURFER_HOME', ''), 'license.txt')
# Precedence: --fs-license-file, $FS_LICENSE, default_license
license_file = opts.fs_license_file or os.getenv('FS_LICENSE',
default_license)
if not os.path.exists(license_file):
raise RuntimeError(
'ERROR: a valid license file is required for FreeSurfer to run. '
'FMRIPREP looked for an existing license file at several paths, in this '
'order: 1) command line argument ``--fs-license-file``; 2) ``$FS_LICENSE`` '
'environment variable; and 3) the ``$FREESURFER_HOME/license.txt`` path. '
'Get it (for free) by registering at https://'
'surfer.nmr.mgh.harvard.edu/registration.html')
os.environ['FS_LICENSE'] = license_file
# Retrieve logging level
log_level = int(max(25 - 5 * opts.verbose_count, logging.DEBUG))
if opts.debug:
log_level = logging.DEBUG
# Set logging
logger.setLevel(log_level)
nlogging.getLogger('workflow').setLevel(log_level)
nlogging.getLogger('interface').setLevel(log_level)
nlogging.getLogger('utils').setLevel(log_level)
errno = 0
# Call build_workflow(opts, retval)
with Manager() as mgr:
retval = mgr.dict()
p = Process(target=build_workflow, args=(opts, retval))
p.start()
p.join()
fmriprep_wf = retval['workflow']
plugin_settings = retval['plugin_settings']
output_dir = retval['output_dir']
work_dir = retval['work_dir']
subject_list = retval['subject_list']
run_uuid = retval['run_uuid']
retcode = retval['return_code']
if fmriprep_wf is None:
sys.exit(1)
if opts.write_graph:
fmriprep_wf.write_graph(graph2use="colored",
format='svg',
simple_form=True)
if opts.reports_only:
sys.exit(int(retcode > 0))
# Clean up master process before running workflow, which may create forks
gc.collect()
try:
fmriprep_wf.run(**plugin_settings)
except RuntimeError as e:
if "Workflow did not execute cleanly" in str(e):
errno = 1
else:
raise
# Generate reports phase
errno += generate_reports(subject_list, output_dir, work_dir, run_uuid)
sys.exit(int(errno > 0))
def main():
"""Entry point"""
from niworkflows.nipype import config as ncfg, logging as nlog
from niworkflows.nipype.pipeline.engine import Workflow
from .. import logging
from ..utils.bids import collect_bids_data
from ..workflows.core import build_workflow
from ..utils.misc import check_folder
# Run parser
opts = get_parser().parse_args()
# Retrieve logging level
log_level = int(max(3 - opts.verbose_count, 0) * 10)
if opts.verbose_count > 1:
log_level = int(max(25 - 5 * opts.verbose_count, 1))
logging.getLogger().setLevel(log_level)
log = logging.getLogger('mriqc.cli')
# Build settings dict
bids_dir = op.abspath(opts.bids_dir)
# Number of processes
n_procs = opts.n_procs
settings = {
'bids_dir': bids_dir,
'write_graph': opts.write_graph,
'testing': opts.testing,
'hmc_afni': opts.hmc_afni,
'hmc_fsl': opts.hmc_fsl,
'fft_spikes_detector': opts.fft_spikes_detector,
'n_procs': n_procs,
'ants_nthreads': opts.ants_nthreads,
'ants_float': opts.ants_float,
'output_dir': op.abspath(opts.output_dir),
'work_dir': op.abspath(opts.work_dir),
'verbose_reports': opts.verbose_reports or opts.testing,
'float32': opts.float32,
'ica': opts.ica,
'no_sub': opts.no_sub,
'email': opts.email,
'fd_thres': opts.fd_thres,
'webapi_url': opts.webapi_url,
'webapi_port': opts.webapi_port,
'upload_strict': opts.upload_strict,
}
if opts.hmc_afni:
settings['deoblique'] = opts.deoblique
settings['despike'] = opts.despike
settings['correct_slice_timing'] = opts.correct_slice_timing
if opts.start_idx:
settings['start_idx'] = opts.start_idx
if opts. stop_idx:
settings['stop_idx'] = opts.stop_idx
if opts.ants_settings:
settings['ants_settings'] = opts.ants_settings
log_dir = op.join(settings['output_dir'], 'logs')
analysis_levels = opts.analysis_level
if opts.participant_label is None:
analysis_levels.append('group')
analysis_levels = list(set(analysis_levels))
if len(analysis_levels) > 2:
raise RuntimeError('Error parsing analysis levels, got "%s"' % ', '.join(analysis_levels))
settings['report_dir'] = opts.report_dir
if not settings['report_dir']:
settings['report_dir'] = op.join(settings['output_dir'], 'reports')
check_folder(settings['output_dir'])
if 'participant' in analysis_levels:
check_folder(settings['work_dir'])
# check_folder(log_dir)
# check_folder(settings['report_dir'])
# Set nipype config
ncfg.update_config({
# 'logging': {'log_directory': log_dir, 'log_to_file': True},
'execution': {#'crashdump_dir': log_dir,
'crashfile_format': 'txt',
'resource_monitor': opts.profile},
})
# Set nipype logging level
nlog.getLogger('workflow').setLevel(log_level)
nlog.getLogger('interface').setLevel(log_level)
nlog.getLogger('utils').setLevel(log_level)
plugin_settings = {'plugin': 'Linear'}
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as pfile:
plugin_settings = loadyml(pfile)
else:
# Setup multiprocessing
if settings['n_procs'] == 0:
settings['n_procs'] = cpu_count()
if settings['ants_nthreads'] == 0:
if settings['n_procs'] > 1:
# always leave one extra thread for non ANTs work,
# don't use more than 8 threads - the speed up is minimal
settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8)
else:
settings['ants_nthreads'] = 1
if settings['n_procs'] > 1:
plugin_settings['plugin'] = 'MultiProc'
plugin_settings['plugin_args'] = {'n_procs': settings['n_procs']}
if opts.mem_gb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb
# Process data types
modalities = opts.modalities
dataset = collect_bids_data(
settings['bids_dir'],
modalities=modalities,
participant_label=opts.participant_label,
session=opts.session_id,
run=opts.run_id,
task=opts.task_id,
)
# Set up participant level
if 'participant' in analysis_levels:
log.info('Participant level started...')
log.info(
'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s',
__version__, ', '.join(analysis_levels), opts.participant_label, settings)
workflow = Workflow(name='workflow_enumerator')
workflow.base_dir = settings['work_dir']
wf_list = []
for mod in modalities:
if not dataset[mod]:
log.warning('No %s scans were found in %s', mod, settings['bids_dir'])
continue
wf_list.append(build_workflow(dataset[mod], mod, settings=settings))
if wf_list:
workflow.add_nodes(wf_list)
if not opts.dry_run:
# Warn about submitting measures BEFORE
if not settings['no_sub']:
log.warning(
'Anonymized quality metrics will be submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
# run MRIQC
workflow.run(**plugin_settings)
# Warn about submitting measures AFTER
if not settings['no_sub']:
log.warning(
'Anonymized quality metrics have beeen submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
else:
msg = 'Error reading BIDS directory ({}), or the dataset is not ' \
'BIDS-compliant.'
if opts.participant_label or opts.session_id or opts.run_id or opts.task_id:
msg = 'The combination of supplied labels'
if opts.participant_label is not None:
msg += ' (--participant_label {})'.format(" ".join(opts.participant_label))
if opts.session_id is not None:
msg += ' (--session-id {})'.format(" ".join(opts.session_id))
if opts.run_id is not None:
msg += ' (--run-id {})'.format(" ".join(opts.run_id))
if opts.task_id is not None:
msg += ' (--task-id {})'.format(" ".join(opts.task_id))
msg += ' did not result in matches within the BIDS directory ({}).'
raise RuntimeError(msg.format(settings['bids_dir']))
log.info('Participant level finished successfully.')
for mod in modalities:
dataframe, order, jsonfiles = generate_csv(settings['output_dir'], mod)
# If there are no iqm.json files, nothing to do.
if dataframe is None:
log.warning(
'No IQM-JSON files were found for the %s data type in %s. The group-level '
'report was not generated.', mod, settings['output_dir'])
continue
if 'jsonfile' in order: order.remove('jsonfile')
base_name = 'mclf_run-20170724-191452_mod-rfc_ver-0.9.7-rc8_class-2_cv-loso'
load_classifier = pkgrf(
'mriqc',
'data/mclf_run-20170724-191452_mod-rfc_ver-0.9.7-rc8_class-2_cv-'
'loso_data-all_estimator.pklz')
cvhelper = CVHelper(load_clf=load_classifier, n_jobs=-1,
rate_label=['rater_1'], basename=base_name)
prediction = cvhelper.predict(dataframe[order])
dataframe['y_prob'] = prediction[0][:,1]
dataframe['y_pred'] = prediction[1]
dataframe.index = jsonfiles
for jsonfile in dataframe.index.values:
with open(jsonfile, 'r+') as json_file:
json_dict = json.load(json_file, object_pairs_hook=OrderedDict)
json_dict['y_prob'] = float(dataframe.get_value(jsonfile, 'y_prob'))
json_dict['y_pred'] = float(dataframe.get_value(jsonfile, 'y_pred'))
json_file.seek(0)
json.dump(json_dict, json_file, separators=(',', ':'), sort_keys=True, indent=4)
json_file.truncate()
def get_ica_confounds(ica_out_dir, ignore_aroma_err):
import os
import shutil
import numpy as np
from niworkflows.nipype import logging
# To catch edge cases when there are either no noise or signal components
LOGGER = logging.getLogger('workflow')
# Pass in numpy array and column base name to generate headers
# modified from add_header_func
def aroma_add_header_func(np_arr, col_base, comp_nums):
import pandas as pd
from sys import version_info
PY3 = version_info[0] > 2
df = pd.DataFrame(
np_arr,
columns=[str(col_base) + str(index) for index in comp_nums])
df.to_csv(str(col_base) + "AROMAConfounds.tsv",
sep="\t" if PY3 else '\t'.encode(),
index=None)
return os.path.abspath(str(col_base) + "AROMAConfounds.tsv")
# load the txt files from ICA_AROMA
melodic_mix = os.path.join(ica_out_dir, 'melodic.ica/melodic_mix')
motion_ics = os.path.join(ica_out_dir, 'classified_motion_ICs.txt')
# Change names of motion_ics and melodic_mix for output
melodic_mix_out = os.path.join(ica_out_dir, 'MELODICmix.tsv')
motion_ics_out = os.path.join(ica_out_dir, 'AROMAnoiseICs.csv')
# melodic_mix replace spaces with tabs
with open(melodic_mix, 'r') as melodic_file:
melodic_mix_out_char = melodic_file.read().replace(' ', '\t')
# write to output file
with open(melodic_mix_out, 'w+') as melodic_file_out:
melodic_file_out.write(melodic_mix_out_char)
# copy metion_ics file to derivatives name
shutil.copyfile(motion_ics, motion_ics_out)
# -1 since python lists start at index 0
motion_ic_indices = np.loadtxt(motion_ics, dtype=int, delimiter=',') - 1
melodic_mix_arr = np.loadtxt(melodic_mix, ndmin=2)
# Return dummy list of ones if no noise compnents were found
if motion_ic_indices.size == 0:
if ignore_aroma_err:
LOGGER.warn('WARNING: No noise components were classified')
aroma_confounds = None
return aroma_confounds, motion_ics_out, melodic_mix_out
else:
raise RuntimeError('ERROR: ICA-AROMA found no noise components!')
# transpose melodic_mix_arr so x refers to the correct dimension
aggr_confounds = np.asarray(
[melodic_mix_arr.T[x] for x in motion_ic_indices])
# the "good" ics, (e.g. not motion related)
good_ic_arr = np.delete(melodic_mix_arr, motion_ic_indices, 1).T
# return dummy lists of zeros if no signal components were found
if good_ic_arr.size == 0:
if ignore_aroma_err:
LOGGER.warn('WARNING: No signal components were classified')
aroma_confounds = None
return aroma_confounds, motion_ics_out, melodic_mix_out
else:
raise RuntimeError('ERROR: ICA-AROMA found no signal components!')
# add one to motion_ic_indices to match melodic report.
aggr_tsv = aroma_add_header_func(
aggr_confounds.T, 'AROMAAggrComp',
[str(x).zfill(2) for x in motion_ic_indices + 1])
aroma_confounds = aggr_tsv
return aroma_confounds, motion_ics_out, melodic_mix_out
def get_ica_confounds(ica_out_dir, ignore_aroma_err):
import os
import shutil
import numpy as np
from niworkflows.nipype import logging
# To catch edge cases when there are either no noise or signal components
LOGGER = logging.getLogger('workflow')
# Pass in numpy array and column base name to generate headers
# modified from add_header_func
def aroma_add_header_func(np_arr, col_base, comp_nums):
import pandas as pd
from sys import version_info
PY3 = version_info[0] > 2
df = pd.DataFrame(np_arr, columns=[str(col_base) + str(index) for index in comp_nums])
df.to_csv(str(col_base) + "AROMAConfounds.tsv",
sep="\t" if PY3 else '\t'.encode(), index=None)
return os.path.abspath(str(col_base) + "AROMAConfounds.tsv")
# load the txt files from ICA_AROMA
melodic_mix = os.path.join(ica_out_dir, 'melodic.ica/melodic_mix')
motion_ics = os.path.join(ica_out_dir, 'classified_motion_ICs.txt')
# Change names of motion_ics and melodic_mix for output
melodic_mix_out = os.path.join(ica_out_dir, 'MELODICmix.tsv')
motion_ics_out = os.path.join(ica_out_dir, 'AROMAnoiseICs.csv')
# melodic_mix replace spaces with tabs
with open(melodic_mix, 'r') as melodic_file:
melodic_mix_out_char = melodic_file.read().replace(' ', '\t')
# write to output file
with open(melodic_mix_out, 'w+') as melodic_file_out:
melodic_file_out.write(melodic_mix_out_char)
# copy metion_ics file to derivatives name
shutil.copyfile(motion_ics, motion_ics_out)
# -1 since python lists start at index 0
motion_ic_indices = np.loadtxt(motion_ics, dtype=int, delimiter=',') - 1
melodic_mix_arr = np.loadtxt(melodic_mix, ndmin=2)
# Return dummy list of ones if no noise compnents were found
if motion_ic_indices.size == 0:
if ignore_aroma_err:
LOGGER.warn('WARNING: No noise components were classified')
aroma_confounds = None
return aroma_confounds, motion_ics_out, melodic_mix_out
else:
raise RuntimeError('ERROR: ICA-AROMA found no noise components!')
# transpose melodic_mix_arr so x refers to the correct dimension
aggr_confounds = np.asarray([melodic_mix_arr.T[x] for x in motion_ic_indices])
# the "good" ics, (e.g. not motion related)
good_ic_arr = np.delete(melodic_mix_arr, motion_ic_indices, 1).T
# return dummy lists of zeros if no signal components were found
if good_ic_arr.size == 0:
if ignore_aroma_err:
LOGGER.warn('WARNING: No signal components were classified')
aroma_confounds = None
return aroma_confounds, motion_ics_out, melodic_mix_out
else:
raise RuntimeError('ERROR: ICA-AROMA found no signal components!')
# add one to motion_ic_indices to match melodic report.
aggr_tsv = aroma_add_header_func(aggr_confounds.T, 'AROMAAggrComp',
[str(x).zfill(2) for x in motion_ic_indices + 1])
aroma_confounds = aggr_tsv
return aroma_confounds, motion_ics_out, melodic_mix_out
def main():
"""Entry point"""
from niworkflows.nipype import config as ncfg, logging as nlog
from niworkflows.nipype.pipeline.engine import Workflow
from .. import logging
from ..utils.bids import collect_bids_data
from ..workflows.core import build_workflow
from ..utils.misc import check_folder
# Run parser
opts = get_parser().parse_args()
# Retrieve logging level
log_level = int(max(3 - opts.verbose_count, 0) * 10)
if opts.verbose_count > 1:
log_level = int(max(25 - 5 * opts.verbose_count, 1))
print(log_level)
logging.getLogger().setLevel(log_level)
log = logging.getLogger('mriqc.cli')
# Build settings dict
bids_dir = op.abspath(opts.bids_dir)
# Number of processes
n_procs = opts.n_procs
settings = {
'bids_dir': bids_dir,
'write_graph': opts.write_graph,
'testing': opts.testing,
'hmc_afni': opts.hmc_afni,
'hmc_fsl': opts.hmc_fsl,
'fft_spikes_detector': opts.fft_spikes_detector,
'n_procs': n_procs,
'ants_nthreads': opts.ants_nthreads,
'output_dir': op.abspath(opts.output_dir),
'work_dir': op.abspath(opts.work_dir),
'verbose_reports': opts.verbose_reports or opts.testing,
'float32': opts.float32,
'ica': opts.ica,
'no_sub': opts.no_sub,
'email': opts.email,
'fd_thres': opts.fd_thres,
'webapi_url' : opts.webapi_url,
'webapi_port' : opts.webapi_port,
'upload_strict' : opts.upload_strict,
}
if opts.hmc_afni:
settings['deoblique'] = opts.deoblique
settings['despike'] = opts.despike
settings['correct_slice_timing'] = opts.correct_slice_timing
if opts.start_idx:
settings['start_idx'] = opts.start_idx
if opts. stop_idx:
settings['stop_idx'] = opts.stop_idx
if opts.ants_settings:
settings['ants_settings'] = opts.ants_settings
log_dir = op.join(settings['output_dir'], 'logs')
analysis_levels = opts.analysis_level
if opts.participant_label is None:
analysis_levels.append('group')
analysis_levels = list(set(analysis_levels))
if len(analysis_levels) > 2:
raise RuntimeError('Error parsing analysis levels, got "%s"' % ', '.join(analysis_levels))
settings['report_dir'] = opts.report_dir
if not settings['report_dir']:
settings['report_dir'] = op.join(settings['output_dir'], 'reports')
check_folder(settings['output_dir'])
if 'participant' in analysis_levels:
check_folder(settings['work_dir'])
check_folder(log_dir)
check_folder(settings['report_dir'])
# Set nipype config
ncfg.update_config({
'logging': {'log_directory': log_dir, 'log_to_file': True},
'execution': {'crashdump_dir': log_dir, 'crashfile_format': 'txt'},
})
# Set nipype logging level
nlog.getLogger('workflow').setLevel(log_level)
nlog.getLogger('interface').setLevel(log_level)
nlog.getLogger('filemanip').setLevel(log_level)
callback_log_path = None
plugin_settings = {'plugin': 'Linear'}
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as pfile:
plugin_settings = loadyml(pfile)
else:
# Setup multiprocessing
if settings['n_procs'] == 0:
settings['n_procs'] = cpu_count()
if settings['ants_nthreads'] == 0:
if settings['n_procs'] > 1:
# always leave one extra thread for non ANTs work,
# don't use more than 8 threads - the speed up is minimal
settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8)
else:
settings['ants_nthreads'] = 1
if settings['n_procs'] > 1:
plugin_settings['plugin'] = 'MultiProc'
plugin_settings['plugin_args'] = {'n_procs': settings['n_procs']}
if opts.mem_gb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb
# Process data types
modalities = opts.modalities
dataset = collect_bids_data(
settings['bids_dir'],
modalities=modalities,
participant_label=opts.participant_label,
session=opts.session_id,
run=opts.run_id,
task=opts.task_id,
)
# Set up participant level
if 'participant' in analysis_levels:
log.info('Participant level started...')
log.info(
'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s',
__version__, ', '.join(analysis_levels), opts.participant_label, settings)
workflow = Workflow(name='workflow_enumerator')
workflow.base_dir = settings['work_dir']
wf_list = []
for mod in modalities:
if not dataset[mod]:
log.warning('No %s scans were found in %s', mod, settings['bids_dir'])
continue
wf_list.append(build_workflow(dataset[mod], mod, settings=settings))
if wf_list:
workflow.add_nodes(wf_list)
if not opts.dry_run:
if plugin_settings['plugin'] == 'MultiProc' and opts.profile:
import logging
from niworkflows.nipype.pipeline.plugins.callback_log import log_nodes_cb
plugin_settings['plugin_args']['status_callback'] = log_nodes_cb
callback_log_path = op.join(log_dir, 'run_stats.log')
logger = logging.getLogger('callback')
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(callback_log_path)
logger.addHandler(handler)
# Warn about submitting measures BEFORE
if not settings['no_sub']:
log.warning(
'Anonymized quality metrics will be submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
# run MRIQC
workflow.run(**plugin_settings)
# Warn about submitting measures AFTER
if not settings['no_sub']:
log.warning(
'Anonymized quality metrics have beeen submitted'
' to MRIQC\'s metrics repository.'
' Use --no-sub to disable submission.')
if callback_log_path is not None:
from niworkflows.nipype.utils.draw_gantt_chart import generate_gantt_chart
generate_gantt_chart(callback_log_path, cores=settings['n_procs'])
else:
msg = """\
Error reading BIDS directory ({}), or the dataset is not \
BIDS-compliant."""
if opts.participant_label is not None:
msg = """\
None of the supplied labels (--participant_label) matched with the \
participants found in the BIDS directory ({})."""
raise RuntimeError(msg.format(settings['bids_dir']))
log.info('Participant level finished successfully.')
# Set up group level
if 'group' in analysis_levels:
from ..reports import group_html
from ..utils.misc import generate_csv # , generate_pred
log.info('Group level started...')
log.info(
'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s',
__version__, ', '.join(analysis_levels), opts.participant_label, settings)
reports_dir = check_folder(op.join(settings['output_dir'], 'reports'))
derivatives_dir = op.join(settings['output_dir'], 'derivatives')
n_group_reports = 0
for mod in modalities:
dataframe, out_csv = generate_csv(derivatives_dir,
settings['output_dir'], mod)
# If there are no iqm.json files, nothing to do.
if dataframe is None:
log.warning(
'No IQM-JSON files were found for the %s data type in %s. The group-level '
'report was not generated.', mod, derivatives_dir)
continue
log.info('Summary CSV table for the %s data generated (%s)', mod, out_csv)
# out_pred = generate_pred(derivatives_dir, settings['output_dir'], mod)
# if out_pred is not None:
# log.info('Predicted QA CSV table for the %s data generated (%s)',
# mod, out_pred)
out_html = op.join(reports_dir, mod + '_group.html')
group_html(out_csv, mod,
csv_failed=op.join(settings['output_dir'], 'failed_' + mod + '.csv'),
out_file=out_html)
log.info('Group-%s report generated (%s)', mod, out_html)
n_group_reports += 1
if n_group_reports == 0:
raise Exception("No data found. No group level reports were generated.")
log.info('Group level finished successfully.')
def main():
"""Entry point"""
from niworkflows.nipype import logging as nlogging
from multiprocessing import set_start_method, Process, Manager
from ..viz.reports import generate_reports
from ..info import __version__
set_start_method('forkserver')
warnings.showwarning = _warn_redirect
opts = get_parser().parse_args()
# FreeSurfer license
default_license = op.join(os.getenv('FREESURFER_HOME', ''), 'license.txt')
# Precedence: --fs-license-file, $FS_LICENSE, default_license
license_file = opts.fs_license_file or os.getenv('FS_LICENSE',
default_license)
if not os.path.exists(license_file):
raise RuntimeError(
'ERROR: a valid license file is required for FreeSurfer to run. '
'FMRIPREP looked for an existing license file at several paths, in this '
'order: 1) command line argument ``--fs-license-file``; 2) ``$FS_LICENSE`` '
'environment variable; and 3) the ``$FREESURFER_HOME/license.txt`` path. '
'Get it (for free) by registering at https://'
'surfer.nmr.mgh.harvard.edu/registration.html')
os.environ['FS_LICENSE'] = license_file
# Retrieve logging level
log_level = int(max(25 - 5 * opts.verbose_count, logging.DEBUG))
# Set logging
logger.setLevel(log_level)
nlogging.getLogger('workflow').setLevel(log_level)
nlogging.getLogger('interface').setLevel(log_level)
nlogging.getLogger('utils').setLevel(log_level)
errno = 0
# Call build_workflow(opts, retval)
with Manager() as mgr:
retval = mgr.dict()
p = Process(target=build_workflow, args=(opts, retval))
p.start()
p.join()
if p.exitcode != 0:
sys.exit(p.exitcode)
fmriprep_wf = retval['workflow']
plugin_settings = retval['plugin_settings']
output_dir = retval['output_dir']
work_dir = retval['work_dir']
subject_list = retval['subject_list']
run_uuid = retval['run_uuid']
retcode = retval['return_code']
if fmriprep_wf is None:
sys.exit(1)
if opts.write_graph:
fmriprep_wf.write_graph(graph2use="colored",
format='svg',
simple_form=True)
if opts.reports_only:
sys.exit(int(retcode > 0))
# Sentry tracking
if not opts.notrack:
try:
from raven import Client
dev_user = bool(int(os.getenv('FMRIPREP_DEV', 0)))
msg = 'fMRIPrep running%s' % (int(dev_user) * ' [dev]')
client = Client(
'https://*****:*****@sentry.io/1137693',
release=__version__)
client.captureMessage(message=msg,
level='debug' if dev_user else 'info',
tags={
'run_id': run_uuid,
'npart': len(subject_list),
'type': 'ping',
'dev': dev_user
})
except Exception:
pass
# Clean up master process before running workflow, which may create forks
gc.collect()
try:
fmriprep_wf.run(**plugin_settings)
except RuntimeError as e:
if "Workflow did not execute cleanly" in str(e):
errno = 1
else:
raise
# Generate reports phase
errno += generate_reports(subject_list, output_dir, work_dir, run_uuid)
sys.exit(int(errno > 0))
