def execute_task(pckld_task, node_config, updatehash):
from socket import gethostname
from traceback import format_exc
from nipype import config, logging
traceback = None
result = None
import os
cwd = os.getcwd()
try:
config.update_config(node_config)
logging.update_logging(config)
from pickle import loads
task = loads(pckld_task)
result = task.run(updatehash=updatehash)
except:
traceback = format_exc()
from pickle import loads
task = loads(pckld_task)
result = task.result
os.chdir(cwd)
return result, traceback, gethostname()
def phantoms_wf(options, cfg):
import glob
import nipype.pipeline.engine as pe
from nipype import config, logging
from nipype.interfaces import utility as niu
from pyacwereg.workflows import evaluation as ev
config.update_config(cfg)
logging.update_logging(config)
grid_size = options.grid_size
if len(grid_size) == 1:
grid_size = grid_size * 3
bs = ev.bspline(name=options.name, shapes=options.shape, snr_list=options.snr, N=options.repetitions)
bs.inputs.inputnode.grid_size = grid_size
bs.inputs.inputnode.lo_matrix = options.lo_matrix
bs.inputs.inputnode.hi_matrix = options.hi_matrix
bs.inputs.inputnode.cortex = options.no_cortex
if options.out_csv is None:
bs.inputs.inputnode.out_csv = op.join(options.work_dir, bs.name, "results.csv")
else:
bs.inputs.inputnode.out_csv = options.out_csv
return bs
def init(cls):
"""Set NiPype configurations."""
from nipype import config as ncfg
# Configure resource_monitor
if cls.resource_monitor:
ncfg.update_config({
"monitoring": {
"enabled": cls.resource_monitor,
"sample_frequency": "0.5",
"summary_append": True,
}
})
ncfg.enable_resource_monitor()
# Nipype config (logs and execution)
ncfg.update_config({
"execution": {
"crashdump_dir": str(execution.log_dir),
"crashfile_format": cls.crashfile_format,
"get_linked_libs": cls.get_linked_libs,
"stop_on_first_crash": cls.stop_on_first_crash,
"check_version": False, # disable future telemetry
}
})
if cls.omp_nthreads is None:
cls.omp_nthreads = min(
cls.nprocs - 1 if cls.nprocs > 1 else os.cpu_count(), 8)
def init(cls):
"""Set NiPype configurations."""
from nipype import config as ncfg
# Configure resource_monitor
if cls.resource_monitor:
ncfg.update_config({
'monitoring': {
'enabled': cls.resource_monitor,
'sample_frequency': '0.5',
'summary_append': True,
}
})
ncfg.enable_resource_monitor()
# Nipype config (logs and execution)
ncfg.update_config({
'execution': {
'crashdump_dir': str(execution.log_dir),
'crashfile_format': cls.crashfile_format,
'get_linked_libs': cls.get_linked_libs,
'stop_on_first_crash': cls.stop_on_first_crash,
}
})
if cls.omp_nthreads is None:
cls.omp_nthreads = min(
cls.nprocs - 1 if cls.nprocs > 1 else os.cpu_count(), 8)
def init(cls):
"""Set NiPype configurations."""
from nipype import config as ncfg
# Configure resource_monitor
if cls.resource_monitor:
ncfg.update_config(
{
"monitoring": {
"enabled": cls.resource_monitor,
"sample_frequency": "0.5",
"summary_append": True,
}
}
)
ncfg.enable_resource_monitor()
# Nipype config (logs and execution)
ncfg.update_config(
{
"execution": {
"crashdump_dir": str(execution.log_dir),
"crashfile_format": cls.crashfile_format,
"get_linked_libs": cls.get_linked_libs,
"stop_on_first_crash": cls.stop_on_first_crash,
"parameterize_dirs": cls.parameterize_dirs,
}
}
)
def init(cls):
"""
Set the log level, initialize all loggers into :py:class:`loggers`.
* Add new logger levels (25: IMPORTANT, and 15: VERBOSE).
* Add a new sub-logger (``cli``).
* Logger configuration.
"""
from nipype import config as ncfg
_handler = logging.StreamHandler(stream=sys.stdout)
_handler.setFormatter(
logging.Formatter(fmt=cls._fmt, datefmt=cls._datefmt))
cls.cli.addHandler(_handler)
cls.default.setLevel(execution.log_level)
cls.cli.setLevel(execution.log_level)
cls.interface.setLevel(execution.log_level)
cls.workflow.setLevel(execution.log_level)
cls.utils.setLevel(execution.log_level)
ncfg.update_config({
"logging": {
"log_directory": str(execution.log_dir),
"log_to_file": True
}
})
def test_init_nibetaseries_participant_wf(
bids_dir, deriv_dir, sub_fmriprep, sub_metadata, bold_file, preproc_file,
sub_events, confounds_file, brainmask_file, atlas_file, atlas_lut,
):
output_dir = op.join(str(bids_dir), 'derivatives', 'atlasCorr')
work_dir = op.join(str(bids_dir), 'derivatives', 'work')
deriv_dir = op.join(str(bids_dir), 'derivatives', 'fmriprep')
ncfg.update_config({
'logging': {'log_directory': work_dir,
'log_to_file': True},
'execution': {'crashdump_dir': work_dir,
'crashfile_format': 'txt',
'parameterize_dirs': False},
})
test_np_wf = init_nibetaseries_participant_wf(
atlas_img=str(atlas_file),
atlas_lut=str(atlas_lut),
bids_dir=str(bids_dir),
derivatives_pipeline_dir=deriv_dir,
exclude_description_label=None,
hrf_model='spm',
high_pass=0.008,
output_dir=output_dir,
run_label=None,
selected_confounds=['WhiteMatter', 'CSF'],
session_label=None,
smoothing_kernel=None,
space_label=None,
subject_list=["01"],
task_label=None,
description_label=None,
work_dir=work_dir)
assert test_np_wf.run()
def ants_ct_wf(subjects_id,
preprocessed_data_dir,
working_dir,
ds_dir,
template_dir,
plugin_name):
import os
from nipype import config
from nipype.pipeline.engine import Node, Workflow, MapNode
import nipype.interfaces.utility as util
import nipype.interfaces.io as nio
from nipype.interfaces.freesurfer.utils import ImageInfo
#####################################
# GENERAL SETTINGS
#####################################
wf = Workflow(name='ants_ct')
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True,
'remove_unnecessary_outputs': True,
'job_finished_timeout': 120})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash')
ds = Node(nio.DataSink(base_directory=ds_dir), name='ds')
#####################################
# GET DATA
#####################################
# GET SUBJECT SPECIFIC STRUCTURAL DATA
in_data_templates = {
't1w': '{subject_id}/raw_niftis/sMRI/t1w_reoriented.nii.gz',
}
in_data = Node(nio.SelectFiles(in_data_templates,
base_directory=preprocessed_data_dir),
name="in_data")
in_data.inputs.subject_id = subjects_id
# GET NKI ANTs templates
ants_templates_templates = {
'brain_template': 'NKI/T_template.nii.gz',
'brain_probability_mask': 'NKI/T_templateProbabilityMask.nii.gz',
'segmentation_priors': 'NKI/Priors/*.nii.gz',
't1_registration_template': 'NKI/T_template_BrainCerebellum.nii.gz'
}
ants_templates = Node(nio.SelectFiles(ants_templates_templates,
base_directory=template_dir),
name="ants_templates")
def process(self):
"""Executes the fMRI pipeline workflow and returns True if successful."""
# Enable the use of the the W3C PROV data model to capture and represent provenance in Nipype
# config.enable_provenance()
# Process time
self.now = datetime.datetime.now().strftime("%Y%m%d_%H%M")
cmp_deriv_subject_directory, nipype_deriv_subject_directory, nipype_fmri_pipeline_subject_dir = \
self.init_subject_derivatives_dirs()
# Initialization
log_file = os.path.join(nipype_fmri_pipeline_subject_dir,
"pypeline.log")
if os.path.isfile(log_file):
os.unlink(log_file)
config.update_config({
"logging": {
"workflow_level": "INFO",
"interface_level": "INFO",
"log_directory": nipype_fmri_pipeline_subject_dir,
"log_to_file": True,
},
"execution": {
"remove_unnecessary_outputs": False,
"stop_on_first_crash": True,
"stop_on_first_rerun": False,
"try_hard_link_datasink": True,
"use_relative_paths": True,
"crashfile_format": "txt",
},
})
logging.update_logging(config)
iflogger = logging.getLogger("nipype.interface")
iflogger.info("**** Processing ****")
flow = self.create_pipeline_flow(
cmp_deriv_subject_directory=cmp_deriv_subject_directory,
nipype_deriv_subject_directory=nipype_deriv_subject_directory,
)
flow.write_graph(graph2use="colored", format="svg", simple_form=False)
# Create dictionary of arguments passed to plugin_args
plugin_args = {
'maxtasksperchild': 1,
'n_procs': self.number_of_cores,
'raise_insufficient': False,
}
flow.run(plugin="MultiProc", plugin_args=plugin_args)
iflogger.info("**** Processing finished ****")
return True
def run_command(self, args):
"""
"""
from tempfile import mkdtemp
from clinica.pipelines.dwi_preprocessing_multi_shell.dwi_preprocessing_multi_shell_pipeline import DwiPreprocessingMultiShell
import os
import errno
from clinica.iotools.utils.data_handling import create_subs_sess_list
from nipype import config
cfg = dict(execution={'parameterize_dirs': False})
config.update_config(cfg)
if args.subjects_sessions_tsv is None:
try:
temp_dir = mkdtemp()
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
create_subs_sess_list(args.bids_directory, temp_dir)
args.subjects_sessions_tsv = os.path.join(temp_dir, 'subjects_sessions_list.tsv')
pipeline = DwiPreprocessingNoddi(
bids_directory=self.absolute_path(args.bids_directory),
caps_directory=self.absolute_path(args.caps_directory),
tsv_file=self.absolute_path(args.subjects_sessions_tsv))
pipeline.parameters = {
'epi_param': dict(
[
('echospacing', args.echo_spacing),
('acc_factor', args.acceleration_factor),
('enc_dir', args.phase_encoding_direction),
('epi_factor', args.epi_factor)
]),
'alt_epi_params': dict(
[
('echospacing', args.echo_spacing),
('acc_factor', args.acceleration_factor),
('enc_dir_alt', args.phase_encoding_direction_alternative),
('epi_factor', args.epi_factor)
])
}
if args.working_directory is None:
args.working_directory = mkdtemp()
pipeline.base_dir = self.absolute_path(args.working_directory)
if args.n_procs:
pipeline.run(plugin='MultiProc',
plugin_args={'n_procs': args.n_procs})
else:
pipeline.run()
def _create_singleSession(dataDict, master_config, interpMode, pipeline_name):
"""
create singleSession workflow on a single session
This is the main function to call when processing a data set with T1 & T2
data. ExperimentBaseDirectoryPrefix is the base of the directory to place results, T1Images & T2Images
are the lists of images to be used in the auto-workup. atlas_fname_wpath is
the path and filename of the atlas to use.
"""
assert 'tissue_classify' in master_config['components'] or \
'auxlmk' in master_config['components'] or \
'denoise' in master_config['components'] or \
'landmark' in master_config['components'] or \
'segmentation' in master_config['components'] or \
'malf_2012_neuro' in master_config['components']
from nipype import config, logging
config.update_config(master_config) # Set universal pipeline options
logging.update_logging(config)
from workflows.baseline import generate_single_session_template_WF
project = dataDict['project']
subject = dataDict['subject']
session = dataDict['session']
blackListFileName = dataDict['T1s'][0] + '_noDenoise'
isBlackList = os.path.isfile(blackListFileName)
pname = "{0}_{1}_{2}".format(master_config['workflow_phase'], subject,
session)
onlyT1 = not (len(dataDict['T2s']) > 0)
sessionWorkflow = generate_single_session_template_WF(
project,
subject,
session,
onlyT1,
master_config,
phase=master_config['workflow_phase'],
interpMode=interpMode,
pipeline_name=pipeline_name,
doDenoise=(not isBlackList))
sessionWorkflow.base_dir = master_config['cachedir']
sessionWorkflow_inputsspec = sessionWorkflow.get_node('inputspec')
sessionWorkflow_inputsspec.inputs.T1s = dataDict['T1s']
sessionWorkflow_inputsspec.inputs.T2s = dataDict['T2s']
sessionWorkflow_inputsspec.inputs.PDs = dataDict['PDs']
sessionWorkflow_inputsspec.inputs.FLs = dataDict['FLs']
sessionWorkflow_inputsspec.inputs.OTHERs = dataDict['OTs']
return sessionWorkflow
def init(cls):
"""Set NiPype configurations."""
from nipype import config as ncfg
# Nipype config (logs and execution)
ncfg.update_config({
"execution": {
"crashdump_dir": str(execution.log_dir),
"crashfile_format": cls.crashfile_format,
"get_linked_libs": cls.get_linked_libs,
"stop_on_first_crash": cls.stop_on_first_crash,
}
})
def ants_ct_wf(subjects_id, preprocessed_data_dir, working_dir, ds_dir,
template_dir, plugin_name):
import os
from nipype import config
from nipype.pipeline.engine import Node, Workflow, MapNode
import nipype.interfaces.utility as util
import nipype.interfaces.io as nio
from nipype.interfaces.freesurfer.utils import ImageInfo
#####################################
# GENERAL SETTINGS
#####################################
wf = Workflow(name='ants_ct')
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'),
execution={
'stop_on_first_crash': True,
'remove_unnecessary_outputs': True,
'job_finished_timeout': 120
})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(
working_dir, 'crash')
ds = Node(nio.DataSink(base_directory=ds_dir), name='ds')
#####################################
# GET DATA
#####################################
# GET SUBJECT SPECIFIC STRUCTURAL DATA
in_data_templates = {
't1w': '{subject_id}/raw_niftis/sMRI/t1w_reoriented.nii.gz',
}
in_data = Node(nio.SelectFiles(in_data_templates,
base_directory=preprocessed_data_dir),
name="in_data")
in_data.inputs.subject_id = subjects_id
# GET NKI ANTs templates
ants_templates_templates = {
'brain_template': 'NKI/T_template.nii.gz',
'brain_probability_mask': 'NKI/T_templateProbabilityMask.nii.gz',
'segmentation_priors': 'NKI/Priors/*.nii.gz',
't1_registration_template': 'NKI/T_template_BrainCerebellum.nii.gz'
}
ants_templates = Node(nio.SelectFiles(ants_templates_templates,
base_directory=template_dir),
name="ants_templates")
def _create_singleSession(dataDict, master_config, interpMode, pipeline_name):
"""
create singleSession workflow on a single session
This is the main function to call when processing a data set with T1 & T2
data. ExperimentBaseDirectoryPrefix is the base of the directory to place results, T1Images & T2Images
are the lists of images to be used in the auto-workup. atlas_fname_wpath is
the path and filename of the atlas to use.
"""
assert 'tissue_classify' in master_config['components'] or \
'auxlmk' in master_config['components'] or \
'denoise' in master_config['components'] or \
'landmark' in master_config['components'] or \
'segmentation' in master_config['components'] or \
'malf_2012_neuro' in master_config['components']
from nipype import config, logging
config.update_config(master_config) # Set universal pipeline options
logging.update_logging(config)
from workflows.baseline import generate_single_session_template_WF
project = dataDict['project']
subject = dataDict['subject']
session = dataDict['session']
blackListFileName = dataDict['T1s'][0] + '_noDenoise'
isBlackList = os.path.isfile(blackListFileName)
pname = "{0}_{1}_{2}".format(master_config['workflow_phase'], subject, session)
onlyT1 = not (len(dataDict['T2s']) > 0)
if onlyT1:
print "T1 Only processing starts ..."
else:
print "Multimodal processing starts ..."
sessionWorkflow = generate_single_session_template_WF(project, subject, session, onlyT1, master_config,
phase=master_config['workflow_phase'],
interpMode=interpMode,
pipeline_name=pipeline_name,
doDenoise=(not isBlackList))
sessionWorkflow.base_dir = master_config['cachedir']
sessionWorkflow_inputsspec = sessionWorkflow.get_node('inputspec')
sessionWorkflow_inputsspec.inputs.T1s = dataDict['T1s']
sessionWorkflow_inputsspec.inputs.T2s = dataDict['T2s']
sessionWorkflow_inputsspec.inputs.PDs = dataDict['PDs']
sessionWorkflow_inputsspec.inputs.FLs = dataDict['FLs']
sessionWorkflow_inputsspec.inputs.OTHERs = dataDict['OTs']
return sessionWorkflow
def RunConnectivityWorkflow(path, outDir, workdir, conf_file=None):
if conf_file is not None:
res = open(conf_file, 'r').read()
conf = json.loads(res)
config.update_config(conf)
plugin = config.get("execution", "plugin")
plugin_args = config.get("execution", "plugin_args")
if plugin_args is not None:
plugin_args = eval(plugin_args)
wf = BuildConnectivityWorkflowSurface(path, outDir)
wf.base_dir = workdir
wf.run(plugin=plugin, plugin_args=plugin_args)
def execute_task(pckld_task, node_config, updatehash):
from socket import gethostname
from traceback import format_exc
from nipype import config, logging
traceback=None
result=None
try:
config.update_config(node_config)
logging.update_logging(config)
from cPickle import loads
task = loads(pckld_task)
result = task.run(updatehash=updatehash)
except:
traceback = format_exc()
result = task.result
return result, traceback, gethostname()
def prep_logging(opts, output_folder):
cli_file = f'{output_folder}/rabies_{opts.rabies_stage}.pkl'
if os.path.isfile(cli_file):
raise ValueError(f"""
A previous run was indicated by the presence of {cli_file}.
This can lead to inconsistencies between previous outputs and the log files.
To prevent this, you are required to manually remove {cli_file}, and we
recommend also removing previous datasinks from the {opts.rabies_stage} RABIES step.
""")
with open(cli_file, 'wb') as handle:
pickle.dump(opts, handle, protocol=pickle.HIGHEST_PROTOCOL)
# remove old versions of the log if already existing
log_path = f'{output_folder}/rabies_{opts.rabies_stage}.log'
if os.path.isfile(log_path):
os.remove(log_path)
config.update_config({'logging': {'log_directory': output_folder,
'log_to_file': True}})
# setting workflow logging level
if opts.verbose==0:
level="WARNING"
elif opts.verbose==1:
level="INFO"
elif opts.verbose<=2:
level="DEBUG"
config.enable_debug_mode()
else:
raise ValueError(f"--verbose must be provided an integer of 0 or above. {opts.verbose} was provided instead.")
# nipype has hard-coded 'nipype.log' filename; we rename it after it is created, and change the handlers
logging.update_logging(config)
os.rename(f'{output_folder}/pypeline.log', log_path)
# change the handlers path to the desired file
for logger in logging.loggers.keys():
log = logging.getLogger(logger)
handler = log.handlers[0]
handler.baseFilename = log_path
# set the defined level of verbose
log = logging.getLogger('nipype.workflow')
log.setLevel(level)
log.debug('Debug ON')
return log
def test_filters_init_nibetaseries_participant_wf(
bids_dir, deriv_dir, sub_fmriprep, sub_metadata, bold_file, preproc_file,
sub_events, confounds_file, brainmask_file,
session_label, task_label, run_label, space_label, description_label):
output_dir = op.join(str(bids_dir), 'derivatives', 'atlasCorr')
work_dir = op.join(str(bids_dir), 'derivatives', 'work')
deriv_dir = op.join(str(bids_dir), 'derivatives', 'fmriprep')
ncfg.update_config({
'logging': {'log_directory': work_dir,
'log_to_file': True},
'execution': {'crashdump_dir': work_dir,
'crashfile_format': 'txt',
'parameterize_dirs': False},
})
with pytest.raises(ValueError) as val_err:
init_nibetaseries_participant_wf(
estimator='lsa',
fir_delays=None,
atlas_img=None,
atlas_lut=None,
bids_dir=str(bids_dir),
database_path=None,
derivatives_pipeline_dir=deriv_dir,
exclude_description_label=None,
hrf_model='spm',
high_pass=0.008,
norm_betas=False,
output_dir=output_dir,
return_residuals=False,
run_label=run_label,
selected_confounds=['white_matter', 'csf'],
session_label=session_label,
signal_scaling=False,
smoothing_kernel=None,
space_label=space_label,
subject_list=["01"],
task_label=task_label,
description_label=description_label,
work_dir=work_dir)
assert "could not find preprocessed outputs:" in str(val_err.value)
def test_valid_init_nibetaseries_participant_wf(
bids_dir, deriv_dir, sub_fmriprep, sub_top_metadata, bold_file, preproc_file,
sub_events, confounds_file, brainmask_file, atlas_file, atlas_lut, bids_db_file,
estimator, fir_delays, hrf_model, signal_scaling, norm_betas):
output_dir = op.join(str(bids_dir), 'derivatives', 'atlasCorr')
work_dir = op.join(str(bids_dir), 'derivatives', 'work')
deriv_dir = op.join(str(bids_dir), 'derivatives', 'fmriprep')
ncfg.update_config({
'logging': {'log_directory': work_dir,
'log_to_file': True},
'execution': {'crashdump_dir': work_dir,
'crashfile_format': 'txt',
'parameterize_dirs': False},
})
test_np_wf = init_nibetaseries_participant_wf(
estimator=estimator,
fir_delays=fir_delays,
atlas_img=str(atlas_file),
atlas_lut=str(atlas_lut),
bids_dir=str(bids_dir),
database_path=str(bids_db_file),
derivatives_pipeline_dir=deriv_dir,
exclude_description_label=None,
hrf_model=hrf_model,
high_pass=0.008,
norm_betas=norm_betas,
output_dir=output_dir,
return_residuals=False,
run_label=None,
selected_confounds=['white_matter', 'csf'],
session_label=None,
signal_scaling=signal_scaling,
smoothing_kernel=None,
space_label=None,
subject_list=["01"],
task_label=None,
description_label=None,
work_dir=work_dir)
assert test_np_wf.run()
def main():
"""Entry point"""
from nipype import config, logging
from regseg.workflows.phantoms import phantoms_wf
options = get_parser().parse_args()
# Setup multiprocessing
nthreads = options.nthreads
if nthreads == 0:
from multiprocessing import cpu_count
nthreads = cpu_count()
cfg = {}
cfg['plugin'] = 'Linear'
if nthreads > 1:
cfg['plugin'] = 'MultiProc'
cfg['plugin_args'] = {'n_proc': nthreads}
# Setup work_dir
if not op.exists(options.work_dir):
os.makedirs(options.work_dir)
# Setup logging dir
log_dir = op.abspath('logs')
cfg['logging'] = {'log_directory': log_dir, 'log_to_file': True}
if not op.exists(log_dir):
os.makedirs(log_dir)
config.update_config(cfg)
logging.update_logging(config)
wf = phantoms_wf(options, cfg)
wf.base_dir = options.work_dir
wf.write_graph(graph2use='hierarchical', format='pdf', simple_form=True)
wf.run()
]
sessions = ['COND']
###############################################################################
wf = pe.Workflow(name='QC')
wf.base_dir = wd
wf.config['execution']['crashdump_dir'] = wf.base_dir + "/crash_files"
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'),
execution={
'stop_on_first_crash': False,
'remove_unnecessary_outputs': False,
'job_finished_timeout': 120
})
config.update_config(nipype_cfg)
#generate distributions per session
subjects2 = [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12',
'13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24',
'25', '26', '27', '28', '29', '30', '31', '32', '33', '34'
]
def get_lists_of_files(data_dir, session, subjects):
tsnr_files = [
data_dir + "/MPP/%s/%s/TEMP/moco_tSNR.nii.gz" % (subject, session)
for subject in subjects
]
log_dir = os.path.join(conf.output_dir, 'logs', 'group_analysis',
resource, 'model_%s' % (conf.model_name))
if not os.path.exists(log_dir):
os.makedirs(log_dir)
else:
print "log_dir already exist"
# enable logging
from nipype import config
from nipype import logging
config.update_config(
{'logging': {
'log_directory': log_dir,
'log_to_file': True
}})
# Temporarily disable until solved
#logging.update_logging(config)
iflogger = logging.getLogger('interface')
''' create the list of paths to all output files to go to model '''
# create the 'ordered_paths' list, which is a list of all of the
# output paths of the output files being included in the current
# group-level analysis model
# 'ordered_paths' is later connected to the 'zmap_files' input
# of the group analysis workflow - the files listed in this list
# are merged into the merged 4D file that goes into group analysis
def process(self):
# Process time
now = datetime.datetime.now().strftime("%Y%m%d_%H%M")
# Initialization
if os.path.exists(os.path.join(self.base_directory,"LOG","pypeline.log")):
os.unlink(os.path.join(self.base_directory,"LOG","pypeline.log"))
config.update_config({'logging': {'log_directory': os.path.join(self.base_directory,"LOG"),
'log_to_file': True},
'execution': {}
})
logging.update_logging(config)
flow = pe.Workflow(name='diffusion_pipeline', base_dir=os.path.join(self.base_directory,'NIPYPE'))
iflogger = logging.getLogger('interface')
# Data import
datasource = pe.Node(interface=nio.DataGrabber(outfields = ['diffusion','T1','T2']), name='datasource')
datasource.inputs.base_directory = os.path.join(self.base_directory,'NIFTI')
datasource.inputs.template = '*'
datasource.inputs.raise_on_empty = False
datasource.inputs.field_template = dict(diffusion=self.global_conf.imaging_model+'.nii.gz',T1='T1.nii.gz',T2='T2.nii.gz')
# Data sinker for output
sinker = pe.Node(nio.DataSink(), name="sinker")
sinker.inputs.base_directory = os.path.join(self.base_directory, "RESULTS")
# Clear previous outputs
self.clear_stages_outputs()
if self.stages['Preprocessing'].enabled:
preproc_flow = self.create_stage_flow("Preprocessing")
flow.connect([
(datasource,preproc_flow,[("diffusion","inputnode.diffusion")]),
])
if self.stages['Segmentation'].enabled:
if self.stages['Segmentation'].config.seg_tool == "Freesurfer":
if self.stages['Segmentation'].config.use_existing_freesurfer_data == False:
self.stages['Segmentation'].config.freesurfer_subjects_dir = os.path.join(self.base_directory)
self.stages['Segmentation'].config.freesurfer_subject_id = os.path.join(self.base_directory,'FREESURFER')
if (not os.path.exists(os.path.join(self.base_directory,'NIPYPE/diffusion_pipeline/segmentation_stage/reconall/result_reconall.pklz'))) and os.path.exists(os.path.join(self.base_directory,'FREESURFER')):
shutil.rmtree(os.path.join(self.base_directory,'FREESURFER'))
seg_flow = self.create_stage_flow("Segmentation")
if self.stages['Segmentation'].config.seg_tool == "Freesurfer":
flow.connect([(datasource,seg_flow, [('T1','inputnode.T1')])])
if self.stages['Parcellation'].enabled:
parc_flow = self.create_stage_flow("Parcellation")
if self.stages['Segmentation'].config.seg_tool == "Freesurfer":
flow.connect([(seg_flow,parc_flow, [('outputnode.subjects_dir','inputnode.subjects_dir'),
('outputnode.subject_id','inputnode.subject_id')]),
])
else:
flow.connect([
(seg_flow,parc_flow,[("outputnode.custom_wm_mask","inputnode.custom_wm_mask")])
])
if self.stages['Registration'].enabled:
reg_flow = self.create_stage_flow("Registration")
flow.connect([
(datasource,reg_flow,[('T1','inputnode.T1'),('T2','inputnode.T2')]),
(preproc_flow,reg_flow, [('outputnode.diffusion_preproc','inputnode.diffusion')]),
(parc_flow,reg_flow, [('outputnode.wm_mask_file','inputnode.wm_mask'),('outputnode.roi_volumes','inputnode.roi_volumes')]),
])
if self.stages['Registration'].config.registration_mode == "BBregister (FS)":
flow.connect([
(seg_flow,reg_flow, [('outputnode.subjects_dir','inputnode.subjects_dir'),
('outputnode.subject_id','inputnode.subject_id')]),
])
if self.stages['Diffusion'].enabled:
diff_flow = self.create_stage_flow("Diffusion")
flow.connect([
(preproc_flow,diff_flow, [('outputnode.diffusion_preproc','inputnode.diffusion')]),
(reg_flow,diff_flow, [('outputnode.wm_mask_registered','inputnode.wm_mask_registered')]),
(reg_flow,diff_flow,[('outputnode.roi_volumes_registered','inputnode.roi_volumes')])
])
if self.stages['Connectome'].enabled:
if self.stages['Diffusion'].config.processing_tool == 'FSL':
self.stages['Connectome'].config.probtrackx = True
con_flow = self.create_stage_flow("Connectome")
flow.connect([
(parc_flow,con_flow, [('outputnode.parcellation_scheme','inputnode.parcellation_scheme')]),
(diff_flow,con_flow, [('outputnode.track_file','inputnode.track_file'),('outputnode.gFA','inputnode.gFA'),
('outputnode.roi_volumes','inputnode.roi_volumes_registered'),
('outputnode.skewness','inputnode.skewness'),('outputnode.kurtosis','inputnode.kurtosis'),
('outputnode.P0','inputnode.P0')]),
(con_flow,sinker, [('outputnode.connectivity_matrices',now+'.connectivity_matrices')])
])
if self.stages['Parcellation'].config.parcellation_scheme == "Custom":
flow.connect([(parc_flow,con_flow, [('outputnode.atlas_info','inputnode.atlas_info')])])
iflogger.info("**** Processing ****")
if(self.number_of_cores != 1):
flow.run(plugin='MultiProc', plugin_args={'n_procs' : self.number_of_cores})
else:
flow.run()
self.fill_stages_outputs()
# Clean undesired folders/files
rm_file_list = ['rh.EC_average','lh.EC_average','fsaverage']
for file_to_rm in rm_file_list:
if os.path.exists(os.path.join(self.base_directory,file_to_rm)):
os.remove(os.path.join(self.base_directory,file_to_rm))
# copy .ini and log file
outdir = os.path.join(self.base_directory,"RESULTS",now)
if not os.path.exists(outdir):
os.makedirs(outdir)
shutil.copy(self.config_file,outdir)
shutil.copy(os.path.join(self.base_directory,'LOG','pypeline.log'),outdir)
iflogger.info("**** Processing finished ****")
return True,'Processing sucessful'
def build_functional_temporal_workflow(resource_pool, config, subject_info, \
run_name, site_name=None):
# build pipeline for each subject, individually
# ~ 5 min 45 sec per subject
# (roughly 345 seconds)
import os
import sys
import nipype.interfaces.io as nio
import nipype.pipeline.engine as pe
import nipype.interfaces.utility as util
import nipype.interfaces.fsl.maths as fsl
import glob
import yaml
import time
from time import strftime
from nipype import config as nyconfig
from nipype import logging
logger = logging.getLogger('workflow')
sub_id = str(subject_info[0])
if subject_info[1]:
session_id = str(subject_info[1])
else:
session_id = "session_0"
if subject_info[2]:
scan_id = str(subject_info[2])
else:
scan_id = "scan_0"
# define and create the output directory
output_dir = os.path.join(config["output_directory"], run_name, \
sub_id, session_id, scan_id)
try:
os.makedirs(output_dir)
except:
if not os.path.isdir(output_dir):
err = "[!] Output directory unable to be created.\n" \
"Path: %s\n\n" % output_dir
raise Exception(err)
else:
pass
log_dir = output_dir
# set up logging
nyconfig.update_config({'logging': {'log_directory': log_dir, 'log_to_file': True}})
logging.update_logging(nyconfig)
# take date+time stamp for run identification purposes
unique_pipeline_id = strftime("%Y%m%d%H%M%S")
pipeline_start_stamp = strftime("%Y-%m-%d_%H:%M:%S")
pipeline_start_time = time.time()
logger.info(pipeline_start_stamp)
logger.info("Contents of resource pool:\n" + str(resource_pool))
logger.info("Configuration settings:\n" + str(config))
# for QAP spreadsheet generation only
config["subject_id"] = sub_id
config["session_id"] = session_id
config["scan_id"] = scan_id
config["run_name"] = run_name
if site_name:
config["site_name"] = site_name
workflow = pe.Workflow(name=scan_id)
workflow.base_dir = os.path.join(config["working_directory"], sub_id, \
session_id)
# set up crash directory
workflow.config['execution'] = \
{'crashdump_dir': config["output_directory"]}
# update that resource pool with what's already in the output directory
for resource in os.listdir(output_dir):
if os.path.isdir(os.path.join(output_dir,resource)) and resource not in resource_pool.keys():
resource_pool[resource] = glob.glob(os.path.join(output_dir, \
resource, "*"))[0]
# resource pool check
invalid_paths = []
for resource in resource_pool.keys():
if not os.path.isfile(resource_pool[resource]):
invalid_paths.append((resource, resource_pool[resource]))
if len(invalid_paths) > 0:
err = "\n\n[!] The paths provided in the subject list to the " \
"following resources are not valid:\n"
for path_tuple in invalid_paths:
err = err + path_tuple[0] + ": " + path_tuple[1] + "\n"
err = err + "\n\n"
raise Exception(err)
# start connecting the pipeline
if "qap_functional_temporal" not in resource_pool.keys():
from qap.qap_workflows import qap_functional_temporal_workflow
workflow, resource_pool = \
qap_functional_temporal_workflow(workflow, resource_pool, config)
# set up the datasinks
new_outputs = 0
if "write_all_outputs" not in config.keys():
config["write_all_outputs"] = False
if config["write_all_outputs"] == True:
for output in resource_pool.keys():
# we use a check for len()==2 here to select those items in the
# resource pool which are tuples of (node, node_output), instead
# of the items which are straight paths to files
# resource pool items which are in the tuple format are the
# outputs that have been created in this workflow because they
# were not present in the subject list YML (the starting resource
# pool) and had to be generated
if len(resource_pool[output]) == 2:
ds = pe.Node(nio.DataSink(), name='datasink_%s' % output)
ds.inputs.base_directory = output_dir
node, out_file = resource_pool[output]
workflow.connect(node, out_file, ds, output)
new_outputs += 1
else:
# write out only the output CSV (default)
output = "qap_functional_temporal"
if len(resource_pool[output]) == 2:
ds = pe.Node(nio.DataSink(), name='datasink_%s' % output)
ds.inputs.base_directory = output_dir
node, out_file = resource_pool[output]
workflow.connect(node, out_file, ds, output)
new_outputs += 1
# run the pipeline (if there is anything to do)
if new_outputs > 0:
workflow.write_graph(dotfilename=os.path.join(output_dir, run_name + \
".dot"), \
simple_form=False)
workflow.run(plugin='MultiProc', plugin_args= \
{'n_procs': config["num_cores_per_subject"]})
else:
print "\nEverything is already done for subject %s." % sub_id
# Remove working directory when done
if config["write_all_outputs"] == False:
try:
work_dir = os.path.join(workflow.base_dir, scan_id)
if os.path.exists(work_dir):
import shutil
shutil.rmtree(work_dir)
except:
print "Couldn\'t remove the working directory!"
pass
pipeline_end_stamp = strftime("%Y-%m-%d_%H:%M:%S")
pipeline_end_time = time.time()
logger.info("Elapsed time (minutes) since last start: %s" \
% ((pipeline_end_time - pipeline_start_time)/60))
logger.info("Pipeline end time: %s" % pipeline_end_stamp)
return workflow
def run_workflow(csv_file, use_pbs, contrasts_name, template):
workflow = pe.Workflow(name='run_level1flow')
workflow.base_dir = os.path.abspath('./workingdirs')
from nipype import config, logging
config.update_config({
'logging': {
'log_directory': os.path.join(workflow.base_dir, 'logs'),
'log_to_file': True,
'workflow_level': 'DEBUG',
'interface_level': 'DEBUG',
}
})
logging.update_logging(config)
config.enable_debug_mode()
# redundant with enable_debug_mode() ...
workflow.stop_on_first_crash = True
workflow.remove_unnecessary_outputs = False
workflow.keep_inputs = True
workflow.hash_method = 'content'
"""
Setup the contrast structure that needs to be evaluated. This is a list of
lists. The inner list specifies the contrasts and has the following format:
[Name,Stat,[list of condition names],[weights on those conditions]. The
condition names must match the `names` listed in the `evt_info` function
described above.
"""
try:
import importlib
mod = importlib.import_module('contrasts.' + contrasts_name)
contrasts = mod.contrasts
# event_names = mod.event_names
except ImportError:
raise RuntimeError('Unknown contrasts: %s. Must exist as a Python'
' module in contrasts directory!' % contrasts_name)
modelfit = create_workflow(contrasts)
import bids_templates as bt
inputnode = pe.Node(niu.IdentityInterface(fields=[
'subject_id',
'session_id',
'run_id',
]),
name='input')
assert csv_file is not None, "--csv argument must be defined!"
reader = niu.CSVReader()
reader.inputs.header = True
reader.inputs.in_file = csv_file
out = reader.run()
subject_list = out.outputs.subject
session_list = out.outputs.session
run_list = out.outputs.run
inputnode.iterables = [
('subject_id', subject_list),
('session_id', session_list),
('run_id', run_list),
]
inputnode.synchronize = True
templates = {
'funcs':
'derivatives/featpreproc/highpassed_files/sub-{subject_id}/ses-{session_id}/func/'
'sub-{subject_id}_ses-{session_id}_*_run-{run_id}*_bold_res-1x1x1_preproc_*.nii.gz',
# 'funcmasks':
# 'featpreproc/func_unwarp/sub-{subject_id}/ses-{session_id}/func/'
# 'sub-{subject_id}_ses-{session_id}_*_run-{run_id}*_bold_res-1x1x1_preproc'
# '_mc_unwarped.nii.gz',
'highpass':
'******'
'sub-{subject_id}_ses-{session_id}_*_run-{run_id}_bold_res-1x1x1_preproc_*.nii.gz',
'motion_parameters':
'derivatives/featpreproc/motion_corrected/sub-{subject_id}/ses-{session_id}/func/'
'sub-{subject_id}_ses-{session_id}_*_run-{run_id}_bold_res-1x1x1_preproc.param.1D',
'motion_outlier_files':
'derivatives/featpreproc/motion_outliers/sub-{subject_id}/ses-{session_id}/func/'
'art.sub-{subject_id}_ses-{session_id}_*_run-{run_id}_bold_res-1x1x1_preproc_mc'
'_maths_outliers.txt',
'event_log':
'sub-{subject_id}/ses-{session_id}/func/'
# 'sub-{subject_id}_ses-{session_id}*_bold_res-1x1x1_preproc'
'sub-{subject_id}_ses-{session_id}*run-{run_id}*'
# '.nii.gz',
'_events.tsv',
'ref_func':
'derivatives/featpreproc/reference/func/*.nii.gz',
'ref_funcmask':
'derivatives/featpreproc/reference/func_mask/*.nii.gz',
}
inputfiles = pe.Node(nio.SelectFiles(templates, base_directory=data_dir),
name='in_files')
workflow.connect([
(inputnode, inputfiles, [
('subject_id', 'subject_id'),
('session_id', 'session_id'),
('run_id', 'run_id'),
]),
])
join_input = pe.JoinNode(
niu.IdentityInterface(fields=[
# 'subject_id',
# 'session_id',
# 'run_id',
'funcs',
'highpass',
'motion_parameters',
'motion_outlier_files',
'event_log',
'ref_func',
'ref_funcmask',
]),
joinsource='input',
joinfield=[
'funcs',
'highpass',
'motion_parameters',
'motion_outlier_files',
'event_log',
],
# unique=True,
name='join_input')
workflow.connect([
(inputfiles, join_input, [
('funcs', 'funcs'),
('highpass', 'highpass'),
('motion_parameters', 'motion_parameters'),
('motion_outlier_files', 'motion_outlier_files'),
('event_log', 'event_log'),
('ref_func', 'ref_func'),
('ref_funcmask', 'ref_funcmask'),
]),
(join_input, modelfit, [
('funcs', 'inputspec.funcs'),
('highpass', 'inputspec.highpass'),
('motion_parameters', 'inputspec.motion_parameters'),
('motion_outlier_files', 'inputspec.motion_outlier_files'),
('event_log', 'inputspec.event_log'),
('ref_func', 'inputspec.ref_func'),
('ref_funcmask', 'inputspec.ref_funcmask'),
]),
])
modelfit.inputs.inputspec.fwhm = 2.0
modelfit.inputs.inputspec.highpass = 50
modelfit.write_graph(simple_form=True)
modelfit.write_graph(graph2use='orig', format='png', simple_form=True)
# modelfit.write_graph(graph2use='detailed', format='png', simple_form=False)
workflow.stop_on_first_crash = True
workflow.keep_inputs = True
workflow.remove_unnecessary_outputs = False
workflow.write_graph(simple_form=True)
workflow.write_graph(graph2use='colored', format='png', simple_form=True)
# workflow.write_graph(graph2use='detailed', format='png', simple_form=False)
if use_pbs:
workflow.run(plugin='PBS',
plugin_args={'template': os.path.expanduser(template)})
else:
workflow.run()
def test_nuisance():
# from CPAC.nuisance import create_nuisance
# cn = create_nuisance()
# subjects_list = open('/home/data/Projects/ADHD200/adhd200_sublist_for_basc_withGSR').readlines()
# subjects_list = [ subject.strip() for subject in subjects_list ]
#
# subject = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/funcpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/func_scale/mapflow/_func_scale0/lfo_3dc_RPI_3dv_3dc_maths.nii.gz'
# csf_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/segpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/_csf_threshold_0.4/seg_mask/mapflow/_seg_mask0/segment_prob_0_flirt_maths_maths_maths.nii.gz'
# wm_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/segpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/_wm_threshold_0.66/seg_mask1/mapflow/_seg_mask10/segment_prob_2_flirt_maths_maths_maths.nii.gz'
# gm_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/segpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/_gm_threshold_0.2/seg_mask2/mapflow/_seg_mask20/segment_prob_1_flirt_maths_maths_maths.nii.gz'
# motion_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/funcpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/func_volreg_1/mapflow/_func_volreg_10/lfo_3dc_RPI_3dv1D.1D'
#
# stor = {'compcor' : True,
# 'wm' : True,
# 'csf' : True,
# 'gm' : True,
# 'global' : True,
# 'pc1' : True,
# 'motion' : True,
# 'linear' : True,
# 'quadratic' : True}
#
# stor2 = {'compcor' : True,
# 'wm' : True,
# 'csf' : True,
# 'gm' : True,
# 'global' : False,
# 'pc1' : True,
# 'motion' : False,
# 'linear' : True,
# 'quadratic' : False}
#
# cn.inputs.inputspec.subject = subject
# cn.inputs.inputspec.gm_mask = gm_file
# cn.inputs.inputspec.wm_mask = wm_file
# cn.inputs.inputspec.csf_mask = csf_file
# cn.inputs.inputspec.motion_components = motion_file
# cn.get_node('residuals').iterables = ('selector',[stor, stor2])
# cn.inputs.inputspec.compcor_ncomponents = 5
#
# cn.run(plugin='MultiProc', plugin_args={'n_procs' : 2})
from nipype import config
config.update_config({'execution': {'remove_unnecessary_outputs':False}})
from CPAC.nuisance import create_nuisance
cn = create_nuisance()
stor = {'compcor' : True,
'wm' : True,
'csf' : True,
'gm' : True,
'global' : True,
'pc1' : True,
'motion' : True,
'linear' : True,
'quadratic' : True}
cn.inputs.inputspec.selector = stor
cn.inputs.inputspec.compcor_ncomponents = 5
cn.inputs.inputspec.motion_components = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/movement_parameters/_scan_rest_1_rest/rest_3dc_RPI_3dv1D.1D'
cn.inputs.inputspec.func_to_anat_linear_xfm = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/functional_to_anat_linear_xfm/_scan_rest_1_rest/rest_3dc_RPI_3dv_3dc_3dT_flirt.mat'
cn.inputs.inputspec.mni_to_anat_linear_xfm = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/mni_to_anatomical_linear_xfm/mprage_RPI_3dc_flirt_inv.mat'
cn.inputs.inputspec.gm_mask = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/anatomical_gm_mask/_gm_threshold_0.2/segment_prob_1_maths_maths_maths.nii.gz'
cn.inputs.inputspec.csf_mask = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/anatomical_csf_mask/_csf_threshold_0.4/segment_prob_0_maths_maths_maths.nii.gz'
cn.inputs.inputspec.wm_mask = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/anatomical_wm_mask/_wm_threshold_0.66/segment_prob_2_maths_maths_maths.nii.gz'
cn.inputs.inputspec.harvard_oxford_mask = '/usr/share/fsl/4.1/data/atlases/HarvardOxford/HarvardOxford-sub-maxprob-thr25-2mm.nii.gz'
cn.inputs.inputspec.subject = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/preprocessed/_scan_rest_1_rest/rest_3dc_RPI_3dv_3dc_maths.nii.gz'
cn.base_dir = '/home/bcheung/cn_run'
def learning_prepare_data_wf(working_dir,
ds_dir,
template_lookup_dict,
behav_file,
qc_file,
in_data_name_list,
data_lookup_dict,
use_n_procs,
plugin_name):
import os
from nipype import config
from nipype.pipeline.engine import Node, Workflow
import nipype.interfaces.utility as util
import nipype.interfaces.io as nio
from prepare_data_utils import vectorize_and_aggregate
from itertools import chain
# ensure in_data_name_list is list of lists
in_data_name_list = [i if type(i) == list else [i] for i in in_data_name_list]
in_data_name_list_unique = list(set(chain.from_iterable(in_data_name_list)))
#####################################
# GENERAL SETTINGS
#####################################
wf = Workflow(name='learning_prepare_data_wf')
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': False,
'remove_unnecessary_outputs': False,
'job_finished_timeout': 120,
'hash_method': 'timestamp'})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash')
ds = Node(nio.DataSink(), name='ds')
ds.inputs.base_directory = os.path.join(ds_dir, 'group_learning_prepare_data')
ds.inputs.regexp_substitutions = [
# ('subject_id_', ''),
('_parcellation_', ''),
('_bp_freqs_', 'bp_'),
('_extraction_method_', ''),
('_subject_id_[A0-9]*/', '')
]
ds_X = Node(nio.DataSink(), name='ds_X')
ds_X.inputs.base_directory = os.path.join(ds_dir, 'vectorized_aggregated_data')
ds_pdf = Node(nio.DataSink(), name='ds_pdf')
ds_pdf.inputs.base_directory = os.path.join(ds_dir, 'pdfs')
ds_pdf.inputs.parameterization = False
#####################################
# SET ITERATORS
#####################################
# SUBJECTS ITERATOR
in_data_name_infosource = Node(util.IdentityInterface(fields=['in_data_name']), name='in_data_name_infosource')
in_data_name_infosource.iterables = ('in_data_name', in_data_name_list_unique)
mulitmodal_in_data_name_infosource = Node(util.IdentityInterface(fields=['multimodal_in_data_name']),
name='mulitmodal_in_data_name_infosource')
mulitmodal_in_data_name_infosource.iterables = ('multimodal_in_data_name', in_data_name_list)
###############################################################################################################
# GET SUBJECTS INFO
# create subjects list based on selection criteria
def create_df_fct(behav_file, qc_file):
import pandas as pd
import os
df = pd.read_pickle(behav_file)
qc = pd.read_pickle(qc_file)
df_all = qc.join(df, how='inner')
assert df_all.index.is_unique, 'duplicates in df index. fix before cont.'
df_all_subjects_pickle_file = os.path.abspath('df_all.pkl')
df_all.to_pickle(df_all_subjects_pickle_file)
full_subjects_list = df_all.index.values
return df_all_subjects_pickle_file, full_subjects_list
create_df = Node(util.Function(input_names=['behav_file', 'qc_file'],
output_names=['df_all_subjects_pickle_file', 'full_subjects_list'],
function=create_df_fct),
name='create_df')
create_df.inputs.behav_file = behav_file
create_df.inputs.qc_file = qc_file
###############################################################################################################
# CREAE FILE LIST
# of files that will be aggregted
def create_file_list_fct(subjects_list, in_data_name, data_lookup_dict, template_lookup_dict):
file_list = []
for s in subjects_list:
file_list.append(data_lookup_dict[in_data_name]['path_str'].format(subject_id=s))
if 'matrix_name' in data_lookup_dict[in_data_name].keys():
matrix_name = data_lookup_dict[in_data_name]['matrix_name']
else:
matrix_name = None
if 'parcellation_path' in data_lookup_dict[in_data_name].keys():
parcellation_path = data_lookup_dict[in_data_name]['parcellation_path']
else:
parcellation_path = None
if 'fwhm' in data_lookup_dict[in_data_name].keys():
fwhm = data_lookup_dict[in_data_name]['fwhm']
if fwhm == 0:
fwhm = None
else:
fwhm = None
if 'mask_name' in data_lookup_dict[in_data_name].keys():
mask_path = template_lookup_dict[data_lookup_dict[in_data_name]['mask_name']]
else:
mask_path = None
if 'use_diagonal' in data_lookup_dict[in_data_name].keys():
use_diagonal = data_lookup_dict[in_data_name]['use_diagonal']
else:
use_diagonal = False
if 'use_fishers_z' in data_lookup_dict[in_data_name].keys():
use_fishers_z = data_lookup_dict[in_data_name]['use_fishers_z']
else:
use_fishers_z = False
if 'df_col_names' in data_lookup_dict[in_data_name].keys():
df_col_names = data_lookup_dict[in_data_name]['df_col_names']
else:
df_col_names = None
return file_list, matrix_name, parcellation_path, fwhm, mask_path, use_diagonal, use_fishers_z, df_col_names
create_file_list = Node(util.Function(input_names=['subjects_list',
'in_data_name',
'data_lookup_dict',
'template_lookup_dict',
],
output_names=['file_list',
'matrix_name',
'parcellation_path',
'fwhm',
'mask_path',
'use_diagonal',
'use_fishers_z',
'df_col_names'],
function=create_file_list_fct),
name='create_file_list')
wf.connect(create_df, 'full_subjects_list', create_file_list, 'subjects_list')
wf.connect(in_data_name_infosource, 'in_data_name', create_file_list, 'in_data_name')
create_file_list.inputs.data_lookup_dict = data_lookup_dict
create_file_list.inputs.template_lookup_dict = template_lookup_dict
###############################################################################################################
# VECTORIZE AND AGGREGATE SUBJECTS
# stack single subject np arrays vertically
vectorize_aggregate_subjects = Node(util.Function(input_names=['in_data_file_list',
'mask_file',
'matrix_name',
'parcellation_path',
'fwhm',
'use_diagonal',
'use_fishers_z',
'df_file',
'df_col_names'],
output_names=['vectorized_aggregated_file',
'unimodal_backprojection_info_file'],
function=vectorize_and_aggregate),
name='vectorize_aggregate_subjects')
wf.connect(create_file_list, 'file_list', vectorize_aggregate_subjects, 'in_data_file_list')
wf.connect(create_file_list, 'mask_path', vectorize_aggregate_subjects, 'mask_file')
wf.connect(create_file_list, 'matrix_name', vectorize_aggregate_subjects, 'matrix_name')
wf.connect(create_file_list, 'parcellation_path', vectorize_aggregate_subjects, 'parcellation_path')
wf.connect(create_file_list, 'fwhm', vectorize_aggregate_subjects, 'fwhm')
wf.connect(create_file_list, 'use_diagonal', vectorize_aggregate_subjects, 'use_diagonal')
wf.connect(create_file_list, 'use_fishers_z', vectorize_aggregate_subjects, 'use_fishers_z')
wf.connect(create_df, 'df_all_subjects_pickle_file', vectorize_aggregate_subjects, 'df_file')
wf.connect(create_file_list, 'df_col_names', vectorize_aggregate_subjects, 'df_col_names')
wf.connect(create_df, 'df_all_subjects_pickle_file', ds_X, 'df_all_subjects_pickle_file')
wf.connect(vectorize_aggregate_subjects, 'vectorized_aggregated_file', ds_X, 'X_file')
wf.connect(vectorize_aggregate_subjects, 'unimodal_backprojection_info_file', ds_X, 'unimodal_backprojection_info_file')
#####################################
# RUN WF
#####################################
wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical')
wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf')
wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf')
if plugin_name == 'CondorDAGMan':
wf.run(plugin=plugin_name)
if plugin_name == 'MultiProc':
wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def downsampel_surfs(subject_id,
working_dir,
freesurfer_dir,
ds_dir,
plugin_name,
use_n_procs):
'''
Workflow resamples e.g. native thickness maps to fsaverage5 space
'''
#####################################
# GENERAL SETTINGS
#####################################
fsl.FSLCommand.set_default_output_type('NIFTI_GZ')
fs.FSCommand.set_default_subjects_dir(freesurfer_dir)
wf = Workflow(name='freesurfer_downsample')
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True,
'remove_unnecessary_outputs': True,
'job_finished_timeout': 15})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash')
ds = Node(nio.DataSink(base_directory=ds_dir), name='ds')
ds.inputs.parameterization = False
#####################
# ITERATORS
#####################
# PARCELLATION ITERATOR
infosource = Node(util.IdentityInterface(fields=['hemi', 'surf_measure', 'fwhm', 'target']), name='infosource')
infosource.iterables = [('hemi', ['lh', 'rh']),
('surf_measure', ['thickness', 'area']),
('fwhm', [0, 5, 10, 20]),
('target', ['fsaverage3', 'fsaverage4', 'fsaverage5']),
]
downsample = Node(fs.model.MRISPreproc(), name='downsample')
downsample.inputs.subjects = [subject_id]
wf.connect(infosource, 'target', downsample, 'target')
wf.connect(infosource, 'hemi', downsample, 'hemi')
wf.connect(infosource, 'surf_measure', downsample, 'surf_measure')
wf.connect(infosource, 'fwhm', downsample, 'fwhm_source')
rename = Node(util.Rename(format_string='%(hemi)s.%(surf_measure)s.%(target)s.%(fwhm)smm'), name='rename')
rename.inputs.keep_ext = True
wf.connect(infosource, 'target', rename, 'target')
wf.connect(infosource, 'hemi', rename, 'hemi')
wf.connect(infosource, 'surf_measure', rename, 'surf_measure')
wf.connect(infosource, 'fwhm', rename, 'fwhm')
wf.connect(downsample, 'out_file', rename, 'in_file')
wf.connect(rename, 'out_file', ds, 'surfs.@surfs')
#
wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical')
wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf')
wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf')
if plugin_name == 'CondorDAGMan':
wf.run(plugin=plugin_name)
if plugin_name == 'MultiProc':
wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def collect_3d_metrics_run_glm_meanRegression(cfg):
import os
from nipype import config
from nipype.pipeline.engine import Node, Workflow, MapNode, JoinNode
import nipype.interfaces.utility as util
import nipype.interfaces.io as nio
import nipype.interfaces.fsl as fsl
import nipype.interfaces.freesurfer as freesurfer
# INPUT PARAMETERS
metrics_data_dir = cfg['metrics_data_dir']
metrics_data_suffix = cfg['metrics_data_suffix']
metric_name = cfg['metric_name']
demos_df = cfg['demos_df']
qc_df = cfg['qc_df']
working_dir = cfg['working_dir']
ds_dir = cfg['ds_dir']
template_dir = cfg['template_dir']
subjects_list = cfg['subjects_list']
use_n_procs = cfg['use_n_procs']
plugin_name = cfg['plugin_name']
#####################################
# GENERAL SETTINGS
#####################################
fsl.FSLCommand.set_default_output_type('NIFTI_GZ')
wf = Workflow(name='LeiCA_collect_metrics')
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True,
'remove_unnecessary_outputs': True,
'job_finished_timeout': 120})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash')
ds = Node(nio.DataSink(base_directory=ds_dir), name='ds')
# get atlas data
templates_atlases = {'GM_mask_MNI_2mm': 'SPM_GM/SPM_GM_mask_2mm.nii.gz',
'GM_mask_MNI_3mm': 'SPM_GM/SPM_GM_mask_3mm.nii.gz',
'brain_mask_MNI_3mm': 'cpac_image_resources/MNI_3mm/MNI152_T1_3mm_brain_mask.nii.gz',
'brain_template_MNI_3mm': 'cpac_image_resources/MNI_3mm/MNI152_T1_3mm.nii.gz'
}
selectfiles_anat_templates = Node(nio.SelectFiles(templates_atlases,
base_directory=template_dir),
name="selectfiles_anat_templates")
# EXPORT SUBJECT LIST
def export_subjects_list_fct(subjects_list):
import os
out_file = os.path.join(os.getcwd(), 'subject_list.txt')
file = open(out_file, 'w')
for subject in subjects_list:
file.write('%s\n'%subject)
file.close()
return out_file
# RESTRICT SUBJECTS LIST TO ADULTS
def get_subjects_list_adults_fct(df_path, df_qc_path, subjects_list):
'''
excludes kids and subjects with missing sex or age
'''
import pandas as pd
import numpy as np
df = pd.read_pickle(df_path)
df_qc = pd.read_pickle(df_qc_path)
df = pd.merge(df, df_qc, left_index=True, right_index=True)
pd.to_pickle(df, 'testdf.pkl')
df['subject_id'] = df.subject_id_x
# fixme exclude subjects with mean_FD>.1
subjects_list_exclude = df[(df.age<18) | (df.mean_FD_Power>.1)].index
subjects_list_adults = subjects_list
for exclude_subject in subjects_list_exclude:
if exclude_subject in subjects_list_adults:
subjects_list_adults.remove(exclude_subject)
missing_info = df[(df.age==999) | ((np.logical_or(df.sex=='M', df.sex=='F'))==False)].index
for missing in missing_info:
if missing in subjects_list_adults:
subjects_list_adults.remove(missing)
# remove subject from subject_list_adults for which no entry exists in df
for subject in subjects_list_adults:
if not(subject in df.index):
subjects_list_adults.remove(subject)
return subjects_list_adults
get_subjects_list_adults = Node(util.Function(input_names=['df_path', 'df_qc_path', 'subjects_list'],
output_names=['subjects_list_adults'],
function=get_subjects_list_adults_fct),
name='get_subjects_list_adults')
get_subjects_list_adults.inputs.df_path = demos_df
get_subjects_list_adults.inputs.df_qc_path = qc_df
get_subjects_list_adults.inputs.subjects_list = subjects_list
export_subjects_list = Node(util.Function(input_names=['subjects_list'],
output_names=['out_file'],
function=export_subjects_list_fct),
name='export_subjects_list')
wf.connect(get_subjects_list_adults, 'subjects_list_adults', export_subjects_list,'subjects_list')
wf.connect(export_subjects_list, 'out_file', ds,'@subjects_list')
# BUILD FILE LIST FOR MERGER
def build_file_list_fct(prefix, subjects_list, suffix):
import os
out_file_list = []
for subject in subjects_list:
out_file_list.append(os.path.join(prefix, subject, suffix))
return out_file_list
build_file_list = Node(util.Function(input_names=['prefix', 'subjects_list', 'suffix'],
output_names=['out_file_list'],
function=build_file_list_fct),
name='build_file_list')
build_file_list.inputs.prefix = metrics_data_dir
wf.connect(get_subjects_list_adults, 'subjects_list_adults', build_file_list,'subjects_list')
build_file_list.inputs.suffix = metrics_data_suffix
# MERGE FILES
merge = Node(fsl.Merge(dimension='t'), name='merge')
wf.connect(build_file_list, 'out_file_list', merge, 'in_files')
merge.inputs.merged_file = metric_name + '_merge.nii.gz'
wf.connect(merge, 'merged_file', ds,'@merge')
# GET MEAN VALUE WITHIN MASK FOR REGRESSION
get_mean_values = Node(fsl.ImageStats(), name='get_mean_values')
get_mean_values.inputs.op_string='-k %s -m'
get_mean_values.inputs.split_4d=True
wf.connect(merge, 'merged_file', get_mean_values, 'in_file')
wf.connect(selectfiles_anat_templates, 'brain_mask_MNI_3mm', get_mean_values, 'mask_file')
# CALC MEAN
mean = Node(fsl.MeanImage(), name='mean')
mean.inputs.out_file = metric_name + '_mean.nii.gz'
wf.connect(merge, 'merged_file', mean, 'in_file')
wf.connect(mean, 'out_file', ds,'@mean')
# CREATE DESIGN FILES
def create_design_files_fct(df_demographics_path, df_qc_path, subjects_list, mean_values):
'''
df_path: df should have columns sex ('M', 'F') & age
function
.restricts df to subjects_list
.creates dummy sex vars, age**2, contrasts
.writes mat & con files
'''
import pandas as pd
import numpy as np
import os
df = pd.read_pickle(df_demographics_path)
df_qc = pd.read_pickle(df_qc_path)
df = pd.merge(df, df_qc, left_index=True, right_index=True)
df['dummy_m'] = (df.sex == 'M').astype('int')
df['dummy_f'] = (df.sex == 'F').astype('int')
df['age2'] = df.age**2
df_use = df.loc[subjects_list]
df_use['mean_values'] = mean_values
#fixme
mat = df_use[['dummy_m', 'dummy_f', 'age','mean_FD_Power']].values
mat_str = [
'/NumWaves %s'%str(mat.shape[1]),
'/NumPoints %s'%str(mat.shape[0]),
'/Matrix'
]
n_cons = 6
cons_str = [
'/ContrastName1 pos_age',
'/ContrastName2 neg_age',
'/ContrastName3 m>f',
'/ContrastName4 f>m',
'/ContrastName5 pos_mean_FD_Power',
'/ContrastName6 neg_mean_FD_Power',
'/NumWaves %s'%str(mat.shape[1]),
'/NumContrasts %s'%str(n_cons),
'',
'/Matrix',
'0 0 1 0',
'0 0 -1 0',
'1 -1 0 0',
'-1 1 0 0',
'0 0 0 1',
'0 0 0 -1'
]
# mat = df_use[['dummy_m', 'dummy_f', 'age', 'age2','mean_FD_Power']].values
#
# mat_str = [
# '/NumWaves %s'%str(mat.shape[1]),
# '/NumPoints %s'%str(mat.shape[0]),
# '/Matrix'
# ]
#
# n_cons = 10
# cons_str = [
# '/ContrastName1 pos_age',
# '/ContrastName2 pos_age2',
# '/ContrastName3 pos_age+age2',
# '/ContrastName4 neg_age',
# '/ContrastName5 neg_age2',
# '/ContrastName6 neg_age+age2',
# '/ContrastName7 m>f',
# '/ContrastName8 f>m',
# '/ContrastName9 pos_mean_FD_Power',
# '/ContrastName10 neg_mean_FD_Power',
# '/NumWaves %s'%str(mat.shape[1]),
# '/NumContrasts %s'%str(n_cons),
# '',
# '/Matrix',
# '0 0 1 0 0',
# '0 0 0 1 0',
# '0 0 .5 .5 0',
# '0 0 -1 0 0',
# '0 0 0 -1 0',
# '0 0 -.5 -.5 0',
# '1 -1 0 0 0',
# '1 1 0 0 0',
# '0 0 0 0 1',
# '0 0 0 0 -1'
# ]
#
mat_file = os.path.join(os.getcwd(), 'design.mat')
mat_file_numerical = os.path.join(os.getcwd(), 'design_mat_num.txt')
con_file = os.path.join(os.getcwd(), 'design.con')
df_used_file = os.path.join(os.getcwd(), 'df_used.csv')
# mat_str_out = ''
# for line in mat_str:
# mat_str_out = mat_str_out[:] + line + '\n'
np.savetxt(mat_file_numerical, mat)
num_file = open(mat_file_numerical, 'r')
nums_str = num_file.read()
num_file.close()
out_file = open(mat_file, 'w')
for line in mat_str:
out_file.write('%s\n' % line)
out_file.write(nums_str)
out_file.close()
out_file = open(con_file, 'w')
for line in cons_str:
out_file.write('%s\n' % line)
out_file.close()
df_use.to_csv(df_used_file)
return mat_file, con_file, df_used_file, n_cons
create_design_files = Node(util.Function(input_names=['df_demographics_path', 'df_qc_path', 'subjects_list', 'mean_values'],
output_names=['mat_file', 'con_file', 'df_used_file', 'n_cons'],
function=create_design_files_fct),
name='create_design_files')
create_design_files.inputs.df_demographics_path = demos_df
create_design_files.inputs.df_qc_path = qc_df
wf.connect(get_subjects_list_adults, 'subjects_list_adults', create_design_files,'subjects_list')
wf.connect(get_mean_values, 'out_stat', create_design_files,'mean_values')
wf.connect(create_design_files,'df_used_file', ds,'glm.@df_used')
smooth = Node(fsl.utils.Smooth(fwhm=4), name='smooth')
wf.connect(merge, 'merged_file', smooth, 'in_file')
def run_randomise_fct(data_file, mat_file, con_file, mask_file):
import os
out_dir = os.path.join(os.getcwd(), 'glm')
os.mkdir(out_dir)
cmd_str = 'randomise -i %s -o glm/glm -d %s -t %s -m %s -n 500 -D -T' %(data_file,mat_file, con_file, mask_file)
file = open('command.txt', 'w')
file.write(cmd_str)
file.close()
os.system(cmd_str)
return out_dir
run_randomise = Node(util.Function(input_names=['data_file', 'mat_file', 'con_file', 'mask_file'],
output_names=['out_dir'],
function=run_randomise_fct),
name='run_randomise')
#fixme
#wf.connect(merge, 'merged_file', run_randomise, 'data_file')
wf.connect(smooth, 'smoothed_file', run_randomise, 'data_file')
wf.connect(create_design_files, 'mat_file', run_randomise, 'mat_file')
wf.connect(create_design_files, 'con_file', run_randomise, 'con_file')
#wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', run_randomise, 'mask_file')
wf.connect(selectfiles_anat_templates, 'brain_mask_MNI_3mm', run_randomise, 'mask_file')
def create_renders_fct(randomise_dir, n_cons, background_img):
import os
thresh = .95
out_files_list = []
corr_list = ['glm_tfce_corrp_tstat', 'glm_tfce_p_tstat']
for corr in corr_list:
for con in range(1,n_cons+1):
output_root = corr + str(con)
in_file = os.path.join(randomise_dir, output_root + '.nii.gz')
out_file = os.path.join(os.getcwd(), 'rendered_' + output_root + '.png')
out_files_list.append(out_file)
cmd_str = 'easythresh %s %s %s %s' % (in_file, str(thresh), background_img, output_root)
file = open('command.txt', 'w')
file.write(cmd_str)
file.close()
os.system(cmd_str)
return out_files_list
create_renders = Node(util.Function(input_names=['randomise_dir', 'n_cons', 'background_img'],
output_names=['out_files_list'],
function=create_renders_fct),
name='create_renders')
wf.connect(run_randomise, 'out_dir', create_renders, 'randomise_dir')
wf.connect(create_design_files, 'n_cons', create_renders, 'n_cons')
wf.connect(selectfiles_anat_templates, 'brain_template_MNI_3mm', create_renders, 'background_img')
wf.connect(create_renders, 'out_files_list', ds, 'glm.@renders')
wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical')
wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf')
wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf')
if plugin_name == 'CondorDAGMan':
wf.run(plugin=plugin_name)
if plugin_name == 'MultiProc':
wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def create_workflow(opts):
import logging
from nipype import config as ncfg
from fmriprep.utils import make_folder
from fmriprep.viz.reports import run_reports
from fmriprep.workflows.base import base_workflow_enumerator
settings = {
'bids_root': op.abspath(opts.bids_dir),
'write_graph': opts.write_graph,
'nthreads': opts.nthreads,
'mem_mb': opts.mem_mb,
'debug': opts.debug,
'ants_nthreads': opts.ants_nthreads,
'skull_strip_ants': opts.skull_strip_ants,
'output_dir': op.abspath(opts.output_dir),
'work_dir': op.abspath(opts.work_dir),
'workflow_type': opts.workflow_type,
'skip_native': opts.skip_native
}
# set up logger
logger = logging.getLogger('cli')
if opts.debug:
settings['ants_t1-mni_settings'] = 't1-mni_registration_test'
logger.setLevel(logging.DEBUG)
log_dir = op.join(settings['output_dir'], 'log')
derivatives = op.join(settings['output_dir'], 'derivatives')
# Check and create output and working directories
# Using make_folder to prevent https://github.com/poldracklab/mriqc/issues/111
make_folder(settings['output_dir'])
make_folder(settings['work_dir'])
make_folder(derivatives)
make_folder(log_dir)
logger.addHandler(logging.FileHandler(op.join(log_dir, 'run_workflow')))
# Set nipype config
ncfg.update_config({
'logging': {'log_directory': log_dir, 'log_to_file': True},
'execution': {'crashdump_dir': log_dir,
'remove_unnecessary_outputs': False}
})
# nipype plugin configuration
plugin_settings = {'plugin': 'Linear'}
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as f:
plugin_settings = loadyml(f)
else:
# Setup multiprocessing
if settings['nthreads'] == 0:
settings['nthreads'] = cpu_count()
if settings['nthreads'] > 1:
plugin_settings['plugin'] = 'MultiProc'
plugin_settings['plugin_args'] = {'n_procs': settings['nthreads']}
if settings['mem_mb']:
plugin_settings['plugin_args']['memory_gb'] = settings['mem_mb']/1024
if settings['ants_nthreads'] == 0:
settings['ants_nthreads'] = cpu_count()
# Determine subjects to be processed
subject_list = opts.participant_label
if subject_list is None or not subject_list:
subject_list = [op.basename(subdir)[4:] for subdir in glob.glob(
op.join(settings['bids_root'], 'sub-*'))]
logger.info('Subject list: %s', ', '.join(subject_list))
# Build main workflow and run
preproc_wf = base_workflow_enumerator(subject_list, task_id=opts.task_id,
settings=settings)
preproc_wf.base_dir = settings['work_dir']
preproc_wf.run(**plugin_settings)
if opts.write_graph:
preproc_wf.write_graph(graph2use="colored", format='svg',
simple_form=True)
run_reports(settings['output_dir'])
def main(args):
subjects, master_config = args
import os
import sys
import traceback
# Set universal pipeline options
from nipype import config
config.update_config(master_config)
assert config.get('execution', 'plugin') == master_config['execution']['plugin']
import nipype.pipeline.engine as pe
import nipype.interfaces.io as nio
from nipype.interfaces.utility import IdentityInterface, Function
import nipype.interfaces.ants as ants
from template import MergeByExtendListElements, xml_filename
from PipeLineFunctionHelpers import mapPosteriorList
from atlasNode import GetAtlasNode, MakeNewAtlasTemplate
from utilities.misc import GenerateSubjectOutputPattern as outputPattern
from utilities.distributed import modify_qsub_args
template = pe.Workflow(name='SubjectAtlas_Template')
template.base_dir = master_config['logging']['log_directory']
if 'previouscache' in master_config:
# Running off previous baseline experiment
BAtlas = GetAtlasNode(master_config['previouscache'], 'BAtlas')
else:
# Running after previous baseline experiment
BAtlas = GetAtlasNode(os.path.dirname(master_config['atlascache']), 'BAtlas')
inputspec = pe.Node(interface=IdentityInterface(fields=['subject']), name='inputspec')
inputspec.iterables = ('subject', subjects)
baselineDG = pe.Node(nio.DataGrabber(infields=['subject'], outfields=['t1_average', 't2_average', 'pd_average',
'fl_average', 'outputLabels', 'posteriorImages']),
name='Baseline_DG')
if 'previousresult' in master_config:
baselineDG.inputs.base_directory = master_config['previousresult']
else:
baselineDG.inputs.base_directory = master_config['resultdir']
baselineDG.inputs.sort_filelist = True
baselineDG.inputs.raise_on_empty = False
baselineDG.inputs.template = '*/%s/*/Baseline/%s.nii.gz'
baselineDG.inputs.template_args['t1_average'] = [['subject', 't1_average_BRAINSABC']]
baselineDG.inputs.template_args['t2_average'] = [['subject', 't2_average_BRAINSABC']]
baselineDG.inputs.template_args['pd_average'] = [['subject', 'pd_average_BRAINSABC']]
baselineDG.inputs.template_args['fl_average'] = [['subject', 'fl_average_BRAINSABC']]
baselineDG.inputs.template_args['outputLabels'] = [['subject', 'brain_label_seg']]
baselineDG.inputs.field_template = {'posteriorImages':'*/%s/*/TissueClassify/POSTERIOR_%s.nii.gz'}
posterior_files = ['AIR', 'BASAL', 'CRBLGM', 'CRBLWM', 'CSF', 'GLOBUS', 'HIPPOCAMPUS', 'NOTCSF', 'NOTGM', 'NOTVB', 'NOTWM',
'SURFGM', 'THALAMUS', 'VB', 'WM']
baselineDG.inputs.template_args['posteriorImages'] = [['subject', posterior_files]]
MergeByExtendListElementsNode = pe.Node(Function(function=MergeByExtendListElements,
input_names=['t1s', 't2s',
'pds', 'fls',
'labels', 'posteriors'],
output_names=['ListOfImagesDictionaries', 'registrationImageTypes',
'interpolationMapping']),
run_without_submitting=True, name="99_MergeByExtendListElements")
from PipeLineFunctionHelpers import WrapPosteriorImagesFromDictionaryFunction as wrapfunc
template.connect([(inputspec, baselineDG, [('subject', 'subject')]),
(baselineDG, MergeByExtendListElementsNode, [('t1_average', 't1s'),
('t2_average', 't2s'),
('pd_average', 'pds'),
('fl_average', 'fls'),
('outputLabels', 'labels'),
(('posteriorImages', wrapfunc), 'posteriors')])
])
myInitAvgWF = pe.Node(interface=ants.AverageImages(), name='Atlas_antsSimpleAverage') # was 'Phase1_antsSimpleAverage'
myInitAvgWF.inputs.dimension = 3
myInitAvgWF.inputs.normalize = True
template.connect(baselineDG, 't1_average', myInitAvgWF, "images")
####################################################################################################
# TEMPLATE_BUILD_RUN_MODE = 'MULTI_IMAGE'
# if numSessions == 1:
# TEMPLATE_BUILD_RUN_MODE = 'SINGLE_IMAGE'
####################################################################################################
from BAWantsRegistrationBuildTemplate import BAWantsRegistrationTemplateBuildSingleIterationWF as registrationWF
buildTemplateIteration1 = registrationWF('iteration01')
# buildTemplateIteration2 = buildTemplateIteration1.clone(name='buildTemplateIteration2')
buildTemplateIteration2 = registrationWF('Iteration02')
MakeNewAtlasTemplateNode = pe.Node(interface=Function(function=MakeNewAtlasTemplate,
input_names=['t1_image', 'deformed_list', 'AtlasTemplate', 'outDefinition'],
output_names=['outAtlasFullPath', 'clean_deformed_list']),
# This is a lot of work, so submit it run_without_submitting=True,
run_without_submitting=True, # HACK: THIS NODE REALLY SHOULD RUN ON THE CLUSTER!
name='99_MakeNewAtlasTemplate')
if master_config['execution']['plugin'] == 'SGE': # for some nodes, the qsub call needs to be modified on the cluster
MakeNewAtlasTemplateNode.plugin_args = {'template': master_config['plugin_args']['template'],
'qsub_args': modify_qsub_args(master_config['queue'], '1000M', 1, 1),
'overwrite': True}
for bt in [buildTemplateIteration1, buildTemplateIteration2]:
##################################################
# *** Hans, is this TODO already addressed? *** #
# ----> # TODO: Change these parameters <---- #
##################################################
BeginANTS = bt.get_node("BeginANTS")
BeginANTS.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'], '9000M', 4, hard=False)}
wimtdeformed = bt.get_node("wimtdeformed")
wimtdeformed.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'], '2000M', 1, 2)}
AvgAffineTransform = bt.get_node("AvgAffineTransform")
AvgAffineTransform.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'], '2000M', 1)}
wimtPassivedeformed = bt.get_node("wimtPassivedeformed")
wimtPassivedeformed.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'], '2000M', 1, 2)}
template.connect([(myInitAvgWF, buildTemplateIteration1, [('output_average_image', 'inputspec.fixed_image')]),
(MergeByExtendListElementsNode, buildTemplateIteration1, [('ListOfImagesDictionaries', 'inputspec.ListOfImagesDictionaries'),
('registrationImageTypes', 'inputspec.registrationImageTypes'),
('interpolationMapping','inputspec.interpolationMapping')]),
(buildTemplateIteration1, buildTemplateIteration2, [('outputspec.template', 'inputspec.fixed_image')]),
(MergeByExtendListElementsNode, buildTemplateIteration2, [('ListOfImagesDictionaries', 'inputspec.ListOfImagesDictionaries'),
('registrationImageTypes','inputspec.registrationImageTypes'),
('interpolationMapping', 'inputspec.interpolationMapping')]),
(inputspec, MakeNewAtlasTemplateNode, [(('subject', xml_filename), 'outDefinition')]),
(BAtlas, MakeNewAtlasTemplateNode, [('ExtendedAtlasDefinition_xml_in', 'AtlasTemplate')]),
(buildTemplateIteration2, MakeNewAtlasTemplateNode, [('outputspec.template', 't1_image'),
('outputspec.passive_deformed_templates', 'deformed_list')]),
])
# Create DataSinks
Atlas_DataSink = pe.Node(nio.DataSink(), name="Atlas_DS")
Atlas_DataSink.overwrite = master_config['ds_overwrite']
Atlas_DataSink.inputs.base_directory = master_config['resultdir']
Subject_DataSink = pe.Node(nio.DataSink(), name="Subject_DS")
Subject_DataSink.overwrite = master_config['ds_overwrite']
Subject_DataSink.inputs.base_directory = master_config['resultdir']
template.connect([(inputspec, Atlas_DataSink, [('subject', 'container')]),
(buildTemplateIteration1, Atlas_DataSink, [('outputspec.template', 'Atlas.iteration1')]), # Unnecessary
(MakeNewAtlasTemplateNode, Atlas_DataSink, [('outAtlasFullPath', 'Atlas.definitions')]),
(BAtlas, Atlas_DataSink, [('template_landmarks_50Lmks_fcsv', 'Atlas.20111119_BCD.@fcsv'),
('template_weights_50Lmks_wts', 'Atlas.20111119_BCD.@wts'),
('LLSModel_50Lmks_hdf5', 'Atlas.20111119_BCD.@hdf5'),
('T1_50Lmks_mdl', 'Atlas.20111119_BCD.@mdl')]),
(inputspec, Subject_DataSink, [(('subject', outputPattern), 'regexp_substitutions')]),
(buildTemplateIteration2, Subject_DataSink, [('outputspec.template', 'ANTSTemplate.@template')]),
(MakeNewAtlasTemplateNode, Subject_DataSink, [('clean_deformed_list', 'ANTSTemplate.@passive_deformed_templates')]),
])
from utils import run_workflow, print_workflow
if False:
print_workflow(template, plugin=master_config['execution']['plugin'], dotfilename='template')
return run_workflow(template, plugin=master_config['execution']['plugin'], plugin_args=master_config['plugin_args'])
def process(self):
# Process time
now = datetime.datetime.now().strftime("%Y%m%d_%H%M")
# Initialization
if os.path.exists(os.path.join(self.base_directory,"LOG","pypeline.log")):
os.unlink(os.path.join(self.base_directory,"LOG","pypeline.log"))
config.update_config({'logging': {'log_directory': os.path.join(self.base_directory,"LOG"),
'log_to_file': True},
'execution': {'remove_unnecessary_outputs': False}
})
logging.update_logging(config)
iflogger = logging.getLogger('interface')
# Data import
datasource = pe.Node(interface=nio.DataGrabber(outfields = ['fMRI','T1','T2']), name='datasource')
datasource.inputs.base_directory = os.path.join(self.base_directory,'NIFTI')
datasource.inputs.template = '*'
datasource.inputs.raise_on_empty = False
datasource.inputs.field_template = dict(fMRI='fMRI.nii.gz',T1='T1.nii.gz',T2='T2.nii.gz')
datasource.inputs.sort_filelist=False
# Data sinker for output
sinker = pe.Node(nio.DataSink(), name="fMRI_sinker")
sinker.inputs.base_directory = os.path.join(self.base_directory, "RESULTS")
# Clear previous outputs
self.clear_stages_outputs()
# Create common_flow
common_flow = self.create_common_flow()
# Create fMRI flow
fMRI_flow = pe.Workflow(name='fMRI_pipeline')
fMRI_inputnode = pe.Node(interface=util.IdentityInterface(fields=["fMRI","T1","T2","subjects_dir","subject_id","wm_mask_file","roi_volumes","wm_eroded","brain_eroded","csf_eroded","parcellation_scheme","atlas_info"]),name="inputnode")
fMRI_outputnode = pe.Node(interface=util.IdentityInterface(fields=["connectivity_matrices"]),name="outputnode")
fMRI_flow.add_nodes([fMRI_inputnode,fMRI_outputnode])
if self.stages['Preprocessing'].enabled:
preproc_flow = self.create_stage_flow("Preprocessing")
fMRI_flow.connect([
(fMRI_inputnode,preproc_flow,[("fMRI","inputnode.functional")]),
])
if self.stages['Registration'].enabled:
reg_flow = self.create_stage_flow("Registration")
fMRI_flow.connect([
(fMRI_inputnode,reg_flow,[('T1','inputnode.T1')]),(fMRI_inputnode,reg_flow,[('T2','inputnode.T2')]),
(preproc_flow,reg_flow, [('outputnode.mean_vol','inputnode.target')]),
(fMRI_inputnode,reg_flow, [('wm_mask_file','inputnode.wm_mask'),('roi_volumes','inputnode.roi_volumes'),
('wm_eroded','inputnode.eroded_wm')])
])
if self.stages['Functional'].config.global_nuisance:
fMRI_flow.connect([
(fMRI_inputnode,reg_flow,[('brain_eroded','inputnode.eroded_brain')])
])
if self.stages['Functional'].config.csf:
fMRI_flow.connect([
(fMRI_inputnode,reg_flow,[('csf_eroded','inputnode.eroded_csf')])
])
if self.stages['Registration'].config.registration_mode == "BBregister (FS)":
fMRI_flow.connect([
(fMRI_inputnode,reg_flow, [('subjects_dir','inputnode.subjects_dir'),
('subject_id','inputnode.subject_id')]),
])
if self.stages['Functional'].enabled:
func_flow = self.create_stage_flow("Functional")
fMRI_flow.connect([
(preproc_flow,func_flow, [('outputnode.functional_preproc','inputnode.preproc_file')]),
(reg_flow,func_flow, [('outputnode.wm_mask_registered','inputnode.registered_wm'),('outputnode.roi_volumes_registered','inputnode.registered_roi_volumes'),
('outputnode.eroded_wm_registered','inputnode.eroded_wm'),('outputnode.eroded_csf_registered','inputnode.eroded_csf'),
('outputnode.eroded_brain_registered','inputnode.eroded_brain')])
])
if self.stages['Functional'].config.scrubbing or self.stages['Functional'].config.motion:
fMRI_flow.connect([
(preproc_flow,func_flow,[("outputnode.par_file","inputnode.motion_par_file")])
])
if self.stages['Connectome'].enabled:
con_flow = self.create_stage_flow("Connectome")
fMRI_flow.connect([
(fMRI_inputnode,con_flow, [('parcellation_scheme','inputnode.parcellation_scheme')]),
(func_flow,con_flow, [('outputnode.func_file','inputnode.func_file'),("outputnode.FD","inputnode.FD"),
("outputnode.DVARS","inputnode.DVARS")]),
(reg_flow,con_flow,[("outputnode.roi_volumes_registered","inputnode.roi_volumes_registered")]),
(con_flow,fMRI_outputnode,[("outputnode.connectivity_matrices","connectivity_matrices")])
])
if self.stages['Parcellation'].config.parcellation_scheme == "Custom":
fMRI_flow.connect([(fMRI_inputnode,con_flow, [('atlas_info','inputnode.atlas_info')])])
# Create NIPYPE flow
flow = pe.Workflow(name='NIPYPE', base_dir=os.path.join(self.base_directory))
flow.connect([
(datasource,common_flow,[("T1","inputnode.T1")]),
(datasource,fMRI_flow,[("fMRI","inputnode.fMRI"),("T1","inputnode.T1"),("T2","inputnode.T2")]),
(common_flow,fMRI_flow,[("outputnode.subjects_dir","inputnode.subjects_dir"),
("outputnode.subject_id","inputnode.subject_id"),
("outputnode.wm_mask_file","inputnode.wm_mask_file"),
("outputnode.roi_volumes","inputnode.roi_volumes"),
("outputnode.wm_eroded","inputnode.wm_eroded"),
("outputnode.brain_eroded","inputnode.brain_eroded"),
("outputnode.csf_eroded","inputnode.csf_eroded"),
("outputnode.parcellation_scheme","inputnode.parcellation_scheme"),
("outputnode.atlas_info","inputnode.atlas_info")]),
(fMRI_flow,sinker,[("outputnode.connectivity_matrices","fMRI.%s.connectivity_matrices"%now)])
])
# Process pipeline
iflogger.info("**** Processing ****")
if(self.number_of_cores != 1):
flow.run(plugin='MultiProc', plugin_args={'n_procs' : self.number_of_cores})
else:
flow.run()
self.fill_stages_outputs()
# Clean undesired folders/files
rm_file_list = ['rh.EC_average','lh.EC_average','fsaverage']
for file_to_rm in rm_file_list:
if os.path.exists(os.path.join(self.base_directory,file_to_rm)):
os.remove(os.path.join(self.base_directory,file_to_rm))
# copy .ini and log file
outdir = os.path.join(self.base_directory,"RESULTS",'fMRI',now)
if not os.path.exists(outdir):
os.makedirs(outdir)
shutil.copy(self.config_file,outdir)
shutil.copy(os.path.join(self.base_directory,'LOG','pypeline.log'),outdir)
iflogger.info("**** Processing finished ****")
return True,'Processing sucessful'
def init_mriqc(opts, retval):
"""Build the workflow enumerator"""
from bids.grabbids import BIDSLayout
from nipype import config as ncfg
from nipype.pipeline.engine import Workflow
from ..utils.bids import collect_bids_data
from ..workflows.core import build_workflow
retval['workflow'] = None
retval['plugin_settings'] = None
# Build settings dict
bids_dir = Path(opts.bids_dir).expanduser()
output_dir = Path(opts.output_dir).expanduser()
# Number of processes
n_procs = opts.n_procs or cpu_count()
settings = {
'bids_dir': bids_dir.resolve(),
'output_dir': output_dir.resolve(),
'work_dir': opts.work_dir.expanduser().resolve(),
'write_graph': opts.write_graph,
'n_procs': n_procs,
'testing': opts.testing,
'hmc_afni': opts.hmc_afni,
'hmc_fsl': opts.hmc_fsl,
'fft_spikes_detector': opts.fft_spikes_detector,
'ants_nthreads': opts.ants_nthreads,
'ants_float': opts.ants_float,
'verbose_reports': opts.verbose_reports or opts.testing,
'float32': opts.float32,
'ica': opts.ica,
'no_sub': opts.no_sub,
'email': opts.email,
'fd_thres': opts.fd_thres,
'webapi_url': opts.webapi_url,
'webapi_port': opts.webapi_port,
'upload_strict': opts.upload_strict,
}
if opts.hmc_afni:
settings['deoblique'] = opts.deoblique
settings['despike'] = opts.despike
settings['correct_slice_timing'] = opts.correct_slice_timing
if opts.start_idx:
settings['start_idx'] = opts.start_idx
if opts. stop_idx:
settings['stop_idx'] = opts.stop_idx
if opts.ants_settings:
settings['ants_settings'] = opts.ants_settings
if opts.dsname:
settings['dataset_name'] = opts.dsname
log_dir = settings['output_dir'] / 'logs'
# Create directories
log_dir.mkdir(parents=True, exist_ok=True)
settings['work_dir'].mkdir(parents=True, exist_ok=True)
# Set nipype config
ncfg.update_config({
'logging': {'log_directory': str(log_dir), 'log_to_file': True},
'execution': {
'crashdump_dir': str(log_dir), 'crashfile_format': 'txt',
'resource_monitor': opts.profile},
})
# Plugin configuration
plugin_settings = {}
if n_procs == 1:
plugin_settings['plugin'] = 'Linear'
if settings['ants_nthreads'] == 0:
settings['ants_nthreads'] = 1
else:
plugin_settings['plugin'] = 'MultiProc'
plugin_settings['plugin_args'] = {'n_procs': n_procs}
if opts.mem_gb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb
if settings['ants_nthreads'] == 0:
# always leave one extra thread for non ANTs work,
# don't use more than 8 threads - the speed up is minimal
settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8)
# Overwrite options if --use-plugin provided
if opts.use_plugin and opts.use_plugin.exists():
from yaml import load as loadyml
with opts.use_plugin.open() as pfile:
plugin_settings.update(loadyml(pfile))
# Process data types
modalities = opts.modalities
layout = BIDSLayout(str(settings['bids_dir']),
exclude=['derivatives', 'sourcedata'])
dataset = collect_bids_data(
layout,
participant_label=opts.participant_label,
session=opts.session_id,
run=opts.run_id,
task=opts.task_id,
bids_type=modalities,
)
workflow = Workflow(name='workflow_enumerator')
workflow.base_dir = settings['work_dir']
wf_list = []
subject_list = []
for mod in modalities:
if dataset[mod]:
wf_list.append(build_workflow(dataset[mod], mod, settings=settings))
subject_list += dataset[mod]
retval['subject_list'] = subject_list
if not wf_list:
retval['return_code'] = 1
return retval
workflow.add_nodes(wf_list)
retval['plugin_settings'] = plugin_settings
retval['workflow'] = workflow
retval['return_code'] = 0
return retval
def calc_local_metrics(brain_mask,
preprocessed_data_dir,
subject_id,
parcellations_dict,
bp_freq_list,
TR,
selectfiles_templates,
working_dir,
ds_dir,
use_n_procs,
plugin_name):
import os
from nipype import config
from nipype.pipeline.engine import Node, Workflow, MapNode
import nipype.interfaces.utility as util
import nipype.interfaces.io as nio
import nipype.interfaces.fsl as fsl
from nipype.interfaces.freesurfer.preprocess import MRIConvert
import CPAC.alff.alff as cpac_alff
import CPAC.reho.reho as cpac_reho
import CPAC.utils.utils as cpac_utils
import utils as calc_metrics_utils
from motion import calculate_FD_P, calculate_FD_J
#####################################
# GENERAL SETTINGS
#####################################
fsl.FSLCommand.set_default_output_type('NIFTI_GZ')
wf = Workflow(name='LeiCA_LIFE_metrics')
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True,
'remove_unnecessary_outputs': True,
'job_finished_timeout': 15})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash')
ds = Node(nio.DataSink(base_directory=ds_dir), name='ds')
ds.inputs.regexp_substitutions = [('MNI_resampled_brain_mask_calc.nii.gz', 'falff.nii.gz'),
('residual_filtered_3dT.nii.gz', 'alff.nii.gz'),
('_parcellation_', ''),
('_bp_freqs_', 'bp_'),
]
#####################
# ITERATORS
#####################
# PARCELLATION ITERATOR
parcellation_infosource = Node(util.IdentityInterface(fields=['parcellation']), name='parcellation_infosource')
parcellation_infosource.iterables = ('parcellation', parcellations_dict.keys())
# BP FILTER ITERATOR
bp_filter_infosource = Node(util.IdentityInterface(fields=['bp_freqs']), name='bp_filter_infosource')
bp_filter_infosource.iterables = ('bp_freqs', bp_freq_list)
selectfiles = Node(nio.SelectFiles(selectfiles_templates,
base_directory=preprocessed_data_dir),
name='selectfiles')
selectfiles.inputs.subject_id = subject_id
# #####################
# # FIX TR IN HEADER
# #####################
# tr_msec = int(TR * 1000)
# tr_str = '-tr %s' % tr_msec
#
# fixed_tr_bp = Node(MRIConvert(out_type='niigz', args=tr_str), name='fixed_tr_bp')
# wf.connect(selectfiles, 'epi_MNI_bp', fixed_tr_bp, 'in_file')
#
# fixed_tr_fullspectrum = Node(MRIConvert(out_type='niigz', args=tr_str), name='fixed_tr_fullspectrum')
# wf.connect(selectfiles, 'epi_MNI_fullspectrum', fixed_tr_fullspectrum, 'in_file')
#####################
# calc FD
#####################
FD_P = Node(util.Function(input_names=['in_file'],
output_names=['FD_ts_file', 'mean_FD_file', 'max_FD_file'],
function=calculate_FD_P),
name='FD_P')
wf.connect(selectfiles, 'moco_parms_file', FD_P, 'in_file')
wf.connect(FD_P, 'FD_ts_file', ds, 'QC.@FD')
wf.connect(FD_P, 'mean_FD_file', ds, 'QC.@mean_FD')
wf.connect(FD_P, 'max_FD_file', ds, 'QC.@max_FD')
FD_J = Node(util.Function(input_names=['in_file'],
output_names=['FD_ts_file', 'mean_FD_file', 'max_FD_file'],
function=calculate_FD_J),
name='FD_J')
wf.connect(selectfiles, 'jenkinson_file', FD_J, 'in_file')
wf.connect(FD_J, 'FD_ts_file', ds, 'QC.@FD_J')
wf.connect(FD_J, 'mean_FD_file', ds, 'QC.@mean_FD_J')
wf.connect(FD_J, 'max_FD_file', ds, 'QC.@max_FD_J')
wf.connect(selectfiles, 'rest2anat_cost_file', ds, 'QC.@cost_file')
#####################
# CALCULATE METRICS
#####################
# f/ALFF
alff = cpac_alff.create_alff('alff')
alff.inputs.hp_input.hp = 0.01
alff.inputs.lp_input.lp = 0.1
alff.inputs.inputspec.rest_mask = brain_mask
#wf.connect(fixed_tr_fullspectrum, 'out_file', alff, 'inputspec.rest_res')
wf.connect(selectfiles, 'epi_MNI_fullspectrum', alff, 'inputspec.rest_res')
wf.connect(alff, 'outputspec.alff_img', ds, 'alff.@alff')
wf.connect(alff, 'outputspec.falff_img', ds, 'alff.@falff')
# f/ALFF_MNI Z-SCORE
alff_z = cpac_utils.get_zscore(input_name='alff', wf_name='alff_z')
alff_z.inputs.inputspec.mask_file = brain_mask
wf.connect(alff, 'outputspec.alff_img', alff_z, 'inputspec.input_file')
wf.connect(alff_z, 'outputspec.z_score_img', ds, 'alff_z.@alff')
falff_z = cpac_utils.get_zscore(input_name='falff', wf_name='falff_z')
falff_z.inputs.inputspec.mask_file = brain_mask
wf.connect(alff, 'outputspec.falff_img', falff_z, 'inputspec.input_file')
wf.connect(falff_z, 'outputspec.z_score_img', ds, 'alff_z.@falff')
# REHO
reho = cpac_reho.create_reho()
reho.inputs.inputspec.cluster_size = 27
reho.inputs.inputspec.rest_mask = brain_mask
#wf.connect(fixed_tr_bp, 'out_file', reho, 'inputspec.rest_res_filt')
wf.connect(selectfiles, 'epi_MNI_BP', reho, 'inputspec.rest_res_filt')
wf.connect(reho, 'outputspec.raw_reho_map', ds, 'reho.@reho')
# VARIABILITY SCORES
variability = Node(util.Function(input_names=['in_file'],
output_names=['out_file'],
function=calc_metrics_utils.calc_variability),
name='variability')
#wf.connect(fixed_tr_bp, 'out_file', variability, 'in_file')
wf.connect(selectfiles, 'epi_MNI_BP', variability, 'in_file')
wf.connect(variability, 'out_file', ds, 'variability.@SD')
variability_z = cpac_utils.get_zscore(input_name='ts_std', wf_name='variability_z')
variability_z.inputs.inputspec.mask_file = brain_mask
wf.connect(variability, 'out_file', variability_z, 'inputspec.input_file')
wf.connect(variability_z, 'outputspec.z_score_img', ds, 'variability_z.@variability_z')
##############
## CON MATS
##############
##############
## extract ts
##############
parcellated_ts = Node(
util.Function(input_names=['in_data', 'parcellation_name', 'parcellations_dict', 'bp_freqs', 'tr'],
output_names=['parcellation_time_series', 'parcellation_time_series_file', 'masker_file'],
function=calc_metrics_utils.extract_parcellation_time_series),
name='parcellated_ts')
parcellated_ts.inputs.parcellations_dict = parcellations_dict
parcellated_ts.inputs.tr = TR
#wf.connect(fixed_tr_fullspectrum, 'out_file', parcellated_ts, 'in_data')
wf.connect(selectfiles, 'epi_MNI_fullspectrum', parcellated_ts, 'in_data')
wf.connect(parcellation_infosource, 'parcellation', parcellated_ts, 'parcellation_name')
wf.connect(bp_filter_infosource, 'bp_freqs', parcellated_ts, 'bp_freqs')
##############
## get conmat
##############
con_mat = Node(util.Function(input_names=['in_data', 'extraction_method'],
output_names=['matrix', 'matrix_file'],
function=calc_metrics_utils.calculate_connectivity_matrix),
name='con_mat')
con_mat.inputs.extraction_method = 'correlation'
wf.connect(parcellated_ts, 'parcellation_time_series', con_mat, 'in_data')
##############
## ds
##############
wf.connect(parcellated_ts, 'parcellation_time_series_file', ds, 'con_mat.parcellated_time_series.@parc_ts')
wf.connect(parcellated_ts, 'masker_file', ds, 'con_mat.parcellated_time_series.@masker')
wf.connect(con_mat, 'matrix_file', ds, 'con_mat.matrix.@mat')
wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical')
wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf')
wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf')
if plugin_name == 'CondorDAGMan':
wf.run(plugin=plugin_name, plugin_args={'initial_specs': 'request_memory = 1500'})
if plugin_name == 'MultiProc':
wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def main():
"""Entry point"""
from nipype import config as ncfg
from nipype.pipeline.engine import Workflow
from mriqc import DEFAULTS
from mriqc.utils.bids import collect_bids_data
from mriqc.workflows.core import build_workflow
# from mriqc.reports.utils import check_reports
parser = ArgumentParser(description='MRI Quality Control',
formatter_class=RawTextHelpFormatter)
parser.add_argument('-v',
'--version',
action='version',
version='mriqc v{}'.format(__version__))
parser.add_argument('bids_dir',
action='store',
help='The directory with the input dataset '
'formatted according to the BIDS standard.')
parser.add_argument(
'output_dir',
action='store',
help='The directory where the output files '
'should be stored. If you are running group level analysis '
'this folder should be prepopulated with the results of the'
'participant level analysis.')
parser.add_argument(
'analysis_level',
action='store',
nargs='+',
help='Level of the analysis that will be performed. '
'Multiple participant level analyses can be run independently '
'(in parallel) using the same output_dir.',
choices=['participant', 'group'])
parser.add_argument(
'--participant_label',
'--subject_list',
'-S',
action='store',
help='The label(s) of the participant(s) that should be analyzed. '
'The label corresponds to sub-<participant_label> from the '
'BIDS spec (so it does not include "sub-"). If this parameter '
'is not provided all subjects should be analyzed. Multiple '
'participants can be specified with a space separated list.',
nargs="*")
g_input = parser.add_argument_group('mriqc specific inputs')
g_input.add_argument('-m',
'--modalities',
action='store',
nargs='*',
choices=['T1w', 'bold', 'T2w'],
default=['T1w', 'bold', 'T2w'])
g_input.add_argument('-s', '--session-id', action='store')
g_input.add_argument('-r', '--run-id', action='store')
g_input.add_argument('--nthreads',
action='store',
type=int,
help='number of threads')
g_input.add_argument('--n_procs',
action='store',
default=0,
type=int,
help='number of threads')
g_input.add_argument('--mem_gb',
action='store',
default=0,
type=int,
help='available total memory')
g_input.add_argument('--write-graph',
action='store_true',
default=False,
help='Write workflow graph.')
g_input.add_argument('--dry-run',
action='store_true',
default=False,
help='Do not run the workflow.')
g_input.add_argument('--use-plugin',
action='store',
default=None,
help='nipype plugin configuration file')
g_input.add_argument('--testing',
action='store_true',
default=False,
help='use testing settings for a minimal footprint')
g_input.add_argument(
'--hmc-afni',
action='store_true',
default=True,
help='Use ANFI 3dvolreg for head motion correction (HMC)')
g_input.add_argument(
'--hmc-fsl',
action='store_true',
default=False,
help='Use FSL MCFLIRT for head motion correction (HMC)')
g_input.add_argument(
'-f',
'--float32',
action='store_true',
default=DEFAULTS['float32'],
help=
"Cast the input data to float32 if it's represented in higher precision "
"(saves space and improves perfomance)")
g_input.add_argument('--fft-spikes-detector',
action='store_true',
default=False,
help='Turn on FFT based spike detector (slow).')
g_outputs = parser.add_argument_group('mriqc specific outputs')
g_outputs.add_argument('-w',
'--work-dir',
action='store',
default=op.join(os.getcwd(), 'work'))
g_outputs.add_argument('--report-dir', action='store')
g_outputs.add_argument('--verbose-reports',
default=False,
action='store_true')
# ANTs options
g_ants = parser.add_argument_group(
'specific settings for ANTs registrations')
g_ants.add_argument(
'--ants-nthreads',
action='store',
type=int,
default=DEFAULTS['ants_nthreads'],
help='number of threads that will be set in ANTs processes')
g_ants.add_argument('--ants-settings',
action='store',
help='path to JSON file with settings for ANTS')
# AFNI head motion correction settings
g_afni = parser.add_argument_group(
'specific settings for AFNI head motion correction')
g_afni.add_argument(
'--deoblique',
action='store_true',
default=False,
help='Deoblique the functional scans during head motion '
'correction preprocessing')
g_afni.add_argument(
'--despike',
action='store_true',
default=False,
help='Despike the functional scans during head motion correction '
'preprocessing')
g_afni.add_argument(
'--start-idx',
action='store',
type=int,
help='Initial volume in functional timeseries that should be '
'considered for preprocessing')
g_afni.add_argument(
'--stop-idx',
action='store',
type=int,
help='Final volume in functional timeseries that should be '
'considered for preprocessing')
g_afni.add_argument('--correct-slice-timing',
action='store_true',
default=False,
help='Perform slice timing correction')
opts = parser.parse_args()
# Build settings dict
bids_dir = op.abspath(opts.bids_dir)
# Number of processes
n_procs = 0
if opts.nthreads is not None:
MRIQC_LOG.warn('Option --nthreads has been deprecated in mriqc 0.8.8. '
'Please use --n_procs instead.')
n_procs = opts.nthreads
if opts.n_procs is not None:
n_procs = opts.n_procs
# Check physical memory
total_memory = opts.mem_gb
if total_memory < 0:
try:
from psutil import virtual_memory
total_memory = virtual_memory().total // (1024**3) + 1
except ImportError:
MRIQC_LOG.warn(
'Total physical memory could not be estimated, using %d'
'GB as default', DEFAULT_MEM_GB)
total_memory = DEFAULT_MEM_GB
if total_memory > 0:
av_procs = total_memory // 4
if av_procs < 1:
MRIQC_LOG.warn(
'Total physical memory is less than 4GB, memory allocation'
' problems are likely to occur.')
n_procs = 1
elif n_procs > av_procs:
n_procs = av_procs
settings = {
'bids_dir': bids_dir,
'write_graph': opts.write_graph,
'testing': opts.testing,
'hmc_afni': opts.hmc_afni,
'hmc_fsl': opts.hmc_fsl,
'fft_spikes_detector': opts.fft_spikes_detector,
'n_procs': n_procs,
'ants_nthreads': opts.ants_nthreads,
'output_dir': op.abspath(opts.output_dir),
'work_dir': op.abspath(opts.work_dir),
'verbose_reports': opts.verbose_reports or opts.testing,
'float32': opts.float32
}
if opts.hmc_afni:
settings['deoblique'] = opts.deoblique
settings['despike'] = opts.despike
settings['correct_slice_timing'] = opts.correct_slice_timing
if opts.start_idx:
settings['start_idx'] = opts.start_idx
if opts.stop_idx:
settings['stop_idx'] = opts.stop_idx
if opts.ants_settings:
settings['ants_settings'] = opts.ants_settings
log_dir = op.join(settings['output_dir'], 'logs')
analysis_levels = opts.analysis_level
if opts.participant_label is None:
analysis_levels.append('group')
analysis_levels = list(set(analysis_levels))
if len(analysis_levels) > 2:
raise RuntimeError('Error parsing analysis levels, got "%s"' %
', '.join(analysis_levels))
settings['report_dir'] = opts.report_dir
if not settings['report_dir']:
settings['report_dir'] = op.join(settings['output_dir'], 'reports')
check_folder(settings['output_dir'])
if 'participant' in analysis_levels:
check_folder(settings['work_dir'])
check_folder(log_dir)
check_folder(settings['report_dir'])
# Set nipype config
ncfg.update_config({
'logging': {
'log_directory': log_dir,
'log_to_file': True
},
'execution': {
'crashdump_dir': log_dir
}
})
plugin_settings = {'plugin': 'Linear'}
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as pfile:
plugin_settings = loadyml(pfile)
else:
# Setup multiprocessing
if settings['n_procs'] == 0:
settings['n_procs'] = 1
max_parallel_ants = cpu_count() // settings['ants_nthreads']
if max_parallel_ants > 1:
settings['n_procs'] = max_parallel_ants
if settings['n_procs'] > 1:
plugin_settings['plugin'] = 'MultiProc'
plugin_settings['plugin_args'] = {'n_procs': settings['n_procs']}
MRIQC_LOG.info(
'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s',
__version__, ', '.join(analysis_levels), opts.participant_label,
settings)
# Process data types
modalities = opts.modalities
dataset = collect_bids_data(settings['bids_dir'],
participant_label=opts.participant_label)
# Set up participant level
if 'participant' in analysis_levels:
workflow = Workflow(name='workflow_enumerator')
workflow.base_dir = settings['work_dir']
wf_list = []
for mod in modalities:
if not dataset[mod]:
MRIQC_LOG.warn('No %s scans were found in %s', mod,
settings['bids_dir'])
continue
wf_list.append(build_workflow(dataset[mod], mod,
settings=settings))
if wf_list:
workflow.add_nodes(wf_list)
if not opts.dry_run:
workflow.run(**plugin_settings)
else:
raise RuntimeError(
'Error reading BIDS directory (%s), or the dataset is not '
'BIDS-compliant.' % settings['bids_dir'])
# Set up group level
if 'group' in analysis_levels:
from mriqc.reports import group_html
from mriqc.utils.misc import generate_csv, generate_pred
reports_dir = check_folder(op.join(settings['output_dir'], 'reports'))
derivatives_dir = op.join(settings['output_dir'], 'derivatives')
n_group_reports = 0
for mod in modalities:
dataframe, out_csv = generate_csv(derivatives_dir,
settings['output_dir'], mod)
# If there are no iqm.json files, nothing to do.
if dataframe is None:
MRIQC_LOG.warn(
'No IQM-JSON files were found for the %s data type in %s. The group-level '
'report was not generated.', mod, derivatives_dir)
continue
MRIQC_LOG.info('Summary CSV table for the %s data generated (%s)',
mod, out_csv)
out_pred = generate_pred(derivatives_dir, settings['output_dir'],
mod)
if out_pred is not None:
MRIQC_LOG.info(
'Predicted QA CSV table for the %s data generated (%s)',
mod, out_pred)
out_html = op.join(reports_dir, mod + '_group.html')
group_html(out_csv,
mod,
csv_failed=op.join(settings['output_dir'],
'failed_' + mod + '.csv'),
out_file=out_html)
MRIQC_LOG.info('Group-%s report generated (%s)', mod, out_html)
n_group_reports += 1
if n_group_reports == 0:
raise Exception(
"No data found. No group level reports were generated.")
def create_workflow(wf_base_dir, input_anat, oasis_path):
'''
Method to create the nipype workflow that is executed for
preprocessing the data
Parameters
----------
wf_base_dir : string
filepath to the base directory to run the workflow
input_anat : string
filepath to the input file to run antsCorticalThickness.sh on
oasis_path : string
filepath to the oasis
Returns
-------
wf : nipype.pipeline.engine.Workflow instance
the workflow to be ran for preprocessing
'''
# Import packages
from act_interface import antsCorticalThickness
import nipype.interfaces.io as nio
import nipype.pipeline.engine as pe
import nipype.interfaces.utility as util
from nipype.interfaces.utility import Function
from nipype import logging as np_logging
from nipype import config
import os
# Init variables
oasis_trt_20 = os.path.join(oasis_path,
'OASIS-TRT-20_jointfusion_DKT31_CMA_labels_in_OASIS-30.nii')
# Setup nipype workflow
if not os.path.exists(wf_base_dir):
os.makedirs(wf_base_dir)
wf = pe.Workflow(name='thickness_workflow')
wf.base_dir = wf_base_dir
# Init log directory
log_dir = wf_base_dir
# Define antsCorticalThickness node
thickness = pe.Node(antsCorticalThickness(), name='thickness')
# Set antsCorticalThickness inputs
thickness.inputs.dimension = 3
thickness.inputs.segmentation_iterations = 1
thickness.inputs.segmentation_weight = 0.25
thickness.inputs.input_skull = input_anat #-a
thickness.inputs.template = oasis_path + 'T_template0.nii.gz' #-e
thickness.inputs.brain_prob_mask = oasis_path + \
'T_template0_BrainCerebellumProbabilityMask.nii.gz' #-m
thickness.inputs.brain_seg_priors = oasis_path + \
'Priors2/priors%d.nii.gz' #-p
thickness.inputs.intensity_template = oasis_path + \
'T_template0_BrainCerebellum.nii.gz' #-t
thickness.inputs.extraction_registration_mask = oasis_path + \
'T_template0_BrainCerebellumExtractionMask.nii.gz' #-f
thickness.inputs.out_prefix = 'OUTPUT_' #-o
thickness.inputs.keep_intermediate_files = 0 #-k
# Node to run ANTs 3dROIStats
ROIstats = pe.Node(util.Function(input_names=['mask','thickness_normd'],
output_names=['roi_stats_file'],
function=roi_func),
name='ROIstats')
wf.connect(thickness, 'cortical_thickness_normalized',
ROIstats, 'thickness_normd')
ROIstats.inputs.mask = oasis_trt_20
# Create datasink node
datasink = pe.Node(nio.DataSink(), name='sinker')
datasink.inputs.base_directory = wf_base_dir
# Connect thickness outputs to datasink
wf.connect(thickness, 'brain_extraction_mask',
datasink, 'output.@brain_extr_mask')
wf.connect(thickness, 'brain_segmentation',
datasink, 'output.@brain_seg')
wf.connect(thickness, 'brain_segmentation_N4',
datasink, 'output.@brain_seg_N4')
wf.connect(thickness, 'brain_segmentation_posteriors_1',
datasink, 'output.@brain_seg_post_1')
wf.connect(thickness, 'brain_segmentation_posteriors_2',
datasink, 'output.@brain_seg_post_2')
wf.connect(thickness, 'brain_segmentation_posteriors_3',
datasink, 'output.@brain_seg_post_3')
wf.connect(thickness, 'brain_segmentation_posteriors_4',
datasink, 'output.@brain_seg_post_4')
wf.connect(thickness, 'brain_segmentation_posteriors_5',
datasink, 'output.@brain_seg_post_5')
wf.connect(thickness, 'brain_segmentation_posteriors_6',
datasink, 'output.@brain_seg_post_6')
wf.connect(thickness, 'cortical_thickness',
datasink, 'output.@cortical_thickness')
wf.connect(thickness, 'cortical_thickness_normalized',
datasink,'output.@cortical_thickness_normalized')
# Connect ROI stats output text file to datasink
wf.connect(ROIstats, 'roi_stats_file', datasink, 'output.@ROIstats')
# Setup crashfile directory and logging
wf.config['execution'] = {'hash_method': 'timestamp',
'crashdump_dir': '/home/ubuntu/crashes'}
config.update_config({'logging': {'log_directory': log_dir,
'log_to_file': True}})
np_logging.update_logging(config)
# Return the workflow
return wf
#-------------#
# Load packages
import nipype.interfaces.io as nio # Data i/o
import nipype.interfaces.fsl as fsl # fsl
import nipype.interfaces.utility as util # utility
import nipype.pipeline.engine as pe # pypeline engine
import os # system functions
import procfunc as func
# config
from nipype import config
cfg = dict(logging=dict(workflow_level = 'DEBUG'),
execution={'stop_on_first_crash': False,
'hash_method': 'timestamp'}) # content
config.update_config(cfg)
# Info source
info = dict(dwi=[['dwi_dirs']],
bvecs=[['dwi_dirs']],
bvals=[['dwi_dirs']])
infosource = pe.Node(interface=util.IdentityInterface(fields=['dwi_dirs']),name='infosource')
infosource.iterables = ('dwi_dirs', dwiDirectories)
# Data grabber DWI
datasource_dwi = pe.Node(interface=nio.DataGrabber(infields=['dwi_dirs'], outfields=info.keys()),
name = 'datasource')
datasource_dwi.inputs.base_directory = dataDirectory
datasource_dwi.inputs.template = '*'
datasource_dwi.inputs.field_template = dict(dwi='%s/*.nii*',
def calc_centrality_metrics(cfg):
import os
from nipype import config
from nipype.pipeline.engine import Node, Workflow
import nipype.interfaces.utility as util
import nipype.interfaces.io as nio
import nipype.interfaces.fsl as fsl
import nipype.interfaces.freesurfer as freesurfer
import CPAC.network_centrality.resting_state_centrality as cpac_centrality
import CPAC.network_centrality.z_score as cpac_centrality_z_score
# INPUT PARAMETERS
dicom_dir = cfg['dicom_dir']
preprocessed_data_dir = cfg['preprocessed_data_dir']
working_dir = cfg['working_dir']
freesurfer_dir = cfg['freesurfer_dir']
template_dir = cfg['template_dir']
script_dir = cfg['script_dir']
ds_dir = cfg['ds_dir']
subjects_list = cfg['subjects_list']
TR_list = cfg['TR_list']
use_n_procs = cfg['use_n_procs']
plugin_name = cfg['plugin_name']
#####################################
# GENERAL SETTINGS
#####################################
fsl.FSLCommand.set_default_output_type('NIFTI_GZ')
freesurfer.FSCommand.set_default_subjects_dir(freesurfer_dir)
wf = Workflow(name='LeiCA_metrics')
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': False,
'remove_unnecessary_outputs': True,
'job_finished_timeout': 120})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash')
ds = Node(nio.DataSink(), name='ds')
ds.inputs.substitutions = [('_TR_id_', 'TR_')]
ds.inputs.regexp_substitutions = [('_subject_id_[A0-9]*/', ''), (
'_z_score[0-9]*/', '')] # , #('dc/_TR_id_[0-9]*/', ''), ('evc/_TR_id_[0-9]*/','')]
#####################################
# SET ITERATORS
#####################################
# GET SCAN TR_ID ITERATOR
scan_infosource = Node(util.IdentityInterface(fields=['TR_id']), name='scan_infosource')
scan_infosource.iterables = ('TR_id', TR_list)
subjects_infosource = Node(util.IdentityInterface(fields=['subject_id']), name='subjects_infosource')
subjects_infosource.iterables = ('subject_id', subjects_list)
def add_subject_id_to_ds_dir_fct(subject_id, ds_path):
import os
out_path = os.path.join(ds_path, subject_id)
return out_path
add_subject_id_to_ds_dir = Node(util.Function(input_names=['subject_id', 'ds_path'],
output_names=['out_path'],
function=add_subject_id_to_ds_dir_fct),
name='add_subject_id_to_ds_dir')
wf.connect(subjects_infosource, 'subject_id', add_subject_id_to_ds_dir, 'subject_id')
add_subject_id_to_ds_dir.inputs.ds_path = ds_dir
wf.connect(add_subject_id_to_ds_dir, 'out_path', ds, 'base_directory')
# get atlas data
templates_atlases = {'GM_mask_MNI_2mm': 'SPM_GM/SPM_GM_mask_2mm.nii.gz',
'GM_mask_MNI_3mm': 'SPM_GM/SPM_GM_mask_3mm.nii.gz',
'FSL_MNI_3mm_template': 'MNI152_T1_3mm_brain.nii.gz',
'vmhc_symm_brain': 'cpac_image_resources/symmetric/MNI152_T1_2mm_brain_symmetric.nii.gz',
'vmhc_symm_brain_3mm': 'cpac_image_resources/symmetric/MNI152_T1_3mm_brain_symmetric.nii.gz',
'vmhc_symm_skull': 'cpac_image_resources/symmetric/MNI152_T1_2mm_symmetric.nii.gz',
'vmhc_symm_brain_mask_dil': 'cpac_image_resources/symmetric/MNI152_T1_2mm_brain_mask_symmetric_dil.nii.gz',
'vmhc_config_file_2mm': 'cpac_image_resources/symmetric/T1_2_MNI152_2mm_symmetric.cnf'
}
selectfiles_anat_templates = Node(nio.SelectFiles(templates_atlases,
base_directory=template_dir),
name="selectfiles_anat_templates")
# GET SUBJECT SPECIFIC FUNCTIONAL AND STRUCTURAL DATA
selectfiles_templates = {
'epi_2_MNI_warp': '{subject_id}/rsfMRI_preprocessing/registration/epi_2_MNI_warp/TR_{TR_id}/*.nii.gz',
'epi_mask': '{subject_id}/rsfMRI_preprocessing/masks/brain_mask_epiSpace/TR_{TR_id}/*.nii.gz',
'preproc_epi_full_spectrum': '{subject_id}/rsfMRI_preprocessing/epis/01_denoised/TR_{TR_id}/*.nii.gz',
'preproc_epi_bp': '{subject_id}/rsfMRI_preprocessing/epis/02_denoised_BP/TR_{TR_id}/*.nii.gz',
'preproc_epi_bp_tNorm': '{subject_id}/rsfMRI_preprocessing/epis/03_denoised_BP_tNorm/TR_{TR_id}/*.nii.gz',
'epi_2_struct_mat': '{subject_id}/rsfMRI_preprocessing/registration/epi_2_struct_mat/TR_{TR_id}/*.mat',
't1w': '{subject_id}/raw_niftis/sMRI/t1w_reoriented.nii.gz',
't1w_brain': '{subject_id}/rsfMRI_preprocessing/struct_prep/t1w_brain/t1w_reoriented_maths.nii.gz',
'epi_bp_tNorm_MNIspace_3mm': '{subject_id}/rsfMRI_preprocessing/epis_MNI_3mm/03_denoised_BP_tNorm/TR_645/residual_filt_norm_warp.nii.gz'
}
selectfiles = Node(nio.SelectFiles(selectfiles_templates,
base_directory=preprocessed_data_dir),
name="selectfiles")
wf.connect(scan_infosource, 'TR_id', selectfiles, 'TR_id')
wf.connect(subjects_infosource, 'subject_id', selectfiles, 'subject_id')
# selectfiles.inputs.subject_id = subject_id
# CREATE TRANSFORMATIONS
# creat MNI 2 epi warp
MNI_2_epi_warp = Node(fsl.InvWarp(), name='MNI_2_epi_warp')
MNI_2_epi_warp.inputs.reference = fsl.Info.standard_image('MNI152_T1_2mm.nii.gz')
wf.connect(selectfiles, 'epi_mask', MNI_2_epi_warp, 'reference')
wf.connect(selectfiles, 'epi_2_MNI_warp', MNI_2_epi_warp, 'warp')
#####################
# CALCULATE METRICS
#####################
# DEGREE
# fixme
# a_mem = 5
# fixme
a_mem = 20
dc = cpac_centrality.create_resting_state_graphs(allocated_memory=a_mem,
wf_name='dc') # allocated_memory = a_mem, wf_name = 'dc')
# dc.plugin_args = {'submit_specs': 'request_memory = 6000'}
# fixme
dc.plugin_args = {'submit_specs': 'request_memory = 20000'}
dc.inputs.inputspec.method_option = 0 # 0 for degree centrality, 1 for eigenvector centrality, 2 for lFCD
dc.inputs.inputspec.threshold_option = 0 # 0 for probability p_value, 1 for sparsity threshold, any other for threshold value
dc.inputs.inputspec.threshold = 0.0001
dc.inputs.inputspec.weight_options = [True,
True] # list of two booleans for binarize and weighted options respectively
wf.connect(selectfiles, 'epi_bp_tNorm_MNIspace_3mm', dc, 'inputspec.subject')
wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', dc, 'inputspec.template')
wf.connect(dc, 'outputspec.centrality_outputs', ds, 'metrics.centrality.dc.@centrality_outputs')
wf.connect(dc, 'outputspec.correlation_matrix', ds, 'metrics.centrality.dc.@correlation_matrix')
wf.connect(dc, 'outputspec.graph_outputs', ds, 'metrics.centrality.dc.@graph_outputs')
# DC Z-SCORE
dc_Z = cpac_centrality_z_score.get_cent_zscore(wf_name='dc_Z')
wf.connect(dc, 'outputspec.centrality_outputs', dc_Z, 'inputspec.input_file')
wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', dc_Z, 'inputspec.mask_file')
wf.connect(dc_Z, 'outputspec.z_score_img', ds, 'metrics.centrality.dc_z.@output')
a_mem = 20
evc = cpac_centrality.create_resting_state_graphs(allocated_memory=a_mem, wf_name='evc')
evc.plugin_args = {'submit_specs': 'request_memory = 20000'}
evc.inputs.inputspec.method_option = 1 # 0 for degree centrality, 1 for eigenvector centrality, 2 for lFCD
evc.inputs.inputspec.threshold_option = 0 # 0 for probability p_value, 1 for sparsity threshold, any other for threshold value
evc.inputs.inputspec.threshold = 0.0001
evc.inputs.inputspec.weight_options = [True,
True] # list of two booleans for binarize and weighted options respectively
wf.connect(selectfiles, 'epi_bp_tNorm_MNIspace_3mm', evc, 'inputspec.subject')
wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', evc, 'inputspec.template')
wf.connect(evc, 'outputspec.centrality_outputs', ds, 'metrics.centrality.evc.@centrality_outputs')
wf.connect(evc, 'outputspec.correlation_matrix', ds, 'metrics.centrality.evc.@correlation_matrix')
wf.connect(evc, 'outputspec.graph_outputs', ds, 'metrics.centrality.evc.@graph_outputs')
# EVC Z-SCORE
evc_Z = cpac_centrality_z_score.get_cent_zscore(wf_name='evc_Z')
wf.connect(evc, 'outputspec.centrality_outputs', evc_Z, 'inputspec.input_file')
wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', evc_Z, 'inputspec.mask_file')
wf.connect(evc_Z, 'outputspec.z_score_img', ds, 'metrics.centrality.evc_z.@output')
wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical')
wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf')
wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf')
if plugin_name == 'CondorDAGMan':
wf.run(plugin=plugin_name)
if plugin_name == 'MultiProc':
wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def build_workflow(opts, retval):
"""
Create the Nipype Workflow that supports the whole execution graph, given the inputs.
All the checks and the construction of the workflow are done
inside this function that has pickleable inputs and output
dictionary (``retval``) to allow isolation using a
``multiprocessing.Process`` that allows smriprep to enforce
a hard-limited memory-scope.
"""
from shutil import copyfile
from os import cpu_count
import uuid
from time import strftime
from subprocess import check_call, CalledProcessError, TimeoutExpired
from pkg_resources import resource_filename as pkgrf
import json
from bids import BIDSLayout
from nipype import logging, config as ncfg
from niworkflows.utils.bids import collect_participants
from ..__about__ import __version__
from ..workflows.base import init_smriprep_wf
logger = logging.getLogger('nipype.workflow')
INIT_MSG = """
Running sMRIPrep version {version}:
* BIDS dataset path: {bids_dir}.
* Participant list: {subject_list}.
* Run identifier: {uuid}.
{spaces}
""".format
# Set up some instrumental utilities
run_uuid = '%s_%s' % (strftime('%Y%m%d-%H%M%S'), uuid.uuid4())
# First check that bids_dir looks like a BIDS folder
bids_dir = opts.bids_dir.resolve()
layout = BIDSLayout(str(bids_dir), validate=False)
subject_list = collect_participants(
layout, participant_label=opts.participant_label)
bids_filters = json.loads(
opts.bids_filter_file.read_text()) if opts.bids_filter_file else None
# Load base plugin_settings from file if --use-plugin
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as f:
plugin_settings = loadyml(f)
plugin_settings.setdefault('plugin_args', {})
else:
# Defaults
plugin_settings = {
'plugin': 'MultiProc',
'plugin_args': {
'raise_insufficient': False,
'maxtasksperchild': 1,
}
}
# Resource management options
# Note that we're making strong assumptions about valid plugin args
# This may need to be revisited if people try to use batch plugins
nprocs = plugin_settings['plugin_args'].get('n_procs')
# Permit overriding plugin config with specific CLI options
if nprocs is None or opts.nprocs is not None:
nprocs = opts.nprocs
if nprocs is None or nprocs < 1:
nprocs = cpu_count()
plugin_settings['plugin_args']['n_procs'] = nprocs
if opts.mem_gb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb
omp_nthreads = opts.omp_nthreads
if omp_nthreads == 0:
omp_nthreads = min(nprocs - 1 if nprocs > 1 else cpu_count(), 8)
if 1 < nprocs < omp_nthreads:
logger.warning(
'Per-process threads (--omp-nthreads=%d) exceed total '
'available CPUs (--nprocs/--ncpus=%d)', omp_nthreads, nprocs)
# Set up directories
output_dir = opts.output_dir.resolve()
log_dir = output_dir / 'smriprep' / 'logs'
work_dir = opts.work_dir.resolve()
# Check and create output and working directories
log_dir.mkdir(parents=True, exist_ok=True)
work_dir.mkdir(parents=True, exist_ok=True)
# Nipype config (logs and execution)
ncfg.update_config({
'logging': {
'log_directory': str(log_dir),
'log_to_file': True
},
'execution': {
'crashdump_dir': str(log_dir),
'crashfile_format': 'txt',
'get_linked_libs': False,
'stop_on_first_crash': opts.stop_on_first_crash,
},
'monitoring': {
'enabled': opts.resource_monitor,
'sample_frequency': '0.5',
'summary_append': True,
}
})
if opts.resource_monitor:
ncfg.enable_resource_monitor()
retval['return_code'] = 0
retval['plugin_settings'] = plugin_settings
retval['bids_dir'] = str(bids_dir)
retval['output_dir'] = str(output_dir)
retval['work_dir'] = str(work_dir)
retval['subject_list'] = subject_list
retval['run_uuid'] = run_uuid
retval['workflow'] = None
# Called with reports only
if opts.reports_only:
from niworkflows.reports import generate_reports
logger.log(25, 'Running --reports-only on participants %s',
', '.join(subject_list))
if opts.run_uuid is not None:
run_uuid = opts.run_uuid
retval['return_code'] = generate_reports(subject_list,
str(output_dir),
run_uuid,
packagename="smriprep")
return retval
logger.log(
25,
INIT_MSG(version=__version__,
bids_dir=bids_dir,
subject_list=subject_list,
uuid=run_uuid,
spaces=opts.output_spaces))
# Build main workflow
retval['workflow'] = init_smriprep_wf(
debug=opts.sloppy,
fast_track=opts.fast_track,
freesurfer=opts.run_reconall,
fs_subjects_dir=opts.fs_subjects_dir,
hires=opts.hires,
layout=layout,
longitudinal=opts.longitudinal,
low_mem=opts.low_mem,
omp_nthreads=omp_nthreads,
output_dir=str(output_dir),
run_uuid=run_uuid,
skull_strip_fixed_seed=opts.skull_strip_fixed_seed,
skull_strip_mode=opts.skull_strip_mode,
skull_strip_template=opts.skull_strip_template[0],
spaces=opts.output_spaces,
subject_list=subject_list,
work_dir=str(work_dir),
bids_filters=bids_filters,
)
retval['return_code'] = 0
boilerplate = retval['workflow'].visit_desc()
(log_dir / 'CITATION.md').write_text(boilerplate)
logger.log(
25, 'Works derived from this sMRIPrep execution should '
'include the following boilerplate:\n\n%s', boilerplate)
# Generate HTML file resolving citations
cmd = [
'pandoc', '-s', '--bibliography',
pkgrf('smriprep',
'data/boilerplate.bib'), '--filter', 'pandoc-citeproc',
'--metadata', 'pagetitle="sMRIPrep citation boilerplate"',
str(log_dir / 'CITATION.md'), '-o',
str(log_dir / 'CITATION.html')
]
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning('Could not generate CITATION.html file:\n%s',
' '.join(cmd))
# Generate LaTex file resolving citations
cmd = [
'pandoc', '-s', '--bibliography',
pkgrf('smriprep', 'data/boilerplate.bib'), '--natbib',
str(log_dir / 'CITATION.md'), '-o',
str(log_dir / 'CITATION.tex')
]
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning('Could not generate CITATION.tex file:\n%s',
' '.join(cmd))
else:
copyfile(pkgrf('smriprep', 'data/boilerplate.bib'),
str(log_dir / 'CITATION.bib'))
return retval
def calc_local_metrics(cfg):
import os
from nipype import config
from nipype.pipeline.engine import Node, Workflow, MapNode
import nipype.interfaces.utility as util
import nipype.interfaces.io as nio
import nipype.interfaces.fsl as fsl
import nipype.interfaces.freesurfer as freesurfer
import CPAC.alff.alff as cpac_alff
import CPAC.reho.reho as cpac_reho
import CPAC.utils.utils as cpac_utils
import CPAC.vmhc.vmhc as cpac_vmhc
import CPAC.registration.registration as cpac_registration
import CPAC.network_centrality.z_score as cpac_centrality_z_score
import utils as calc_metrics_utils
# INPUT PARAMETERS
dicom_dir = cfg['dicom_dir']
preprocessed_data_dir = cfg['preprocessed_data_dir']
working_dir = cfg['working_dir']
freesurfer_dir = cfg['freesurfer_dir']
template_dir = cfg['template_dir']
script_dir = cfg['script_dir']
ds_dir = cfg['ds_dir']
subject_id = cfg['subject_id']
TR_list = cfg['TR_list']
vols_to_drop = cfg['vols_to_drop']
rois_list = cfg['rois_list']
lp_cutoff_freq = cfg['lp_cutoff_freq']
hp_cutoff_freq = cfg['hp_cutoff_freq']
use_fs_brainmask = cfg['use_fs_brainmask']
use_n_procs = cfg['use_n_procs']
plugin_name = cfg['plugin_name']
#####################################
# GENERAL SETTINGS
#####################################
fsl.FSLCommand.set_default_output_type('NIFTI_GZ')
freesurfer.FSCommand.set_default_subjects_dir(freesurfer_dir)
wf = Workflow(name='LeiCA_metrics')
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True,
'remove_unnecessary_outputs': True,
'job_finished_timeout': 120})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash')
ds = Node(nio.DataSink(base_directory=ds_dir), name='ds')
ds.inputs.substitutions = [('_TR_id_', 'TR_')]
ds.inputs.regexp_substitutions = [('_variabilty_MNIspace_3mm[0-9]*/', ''), ('_z_score[0-9]*/', '')]
#####################################
# SET ITERATORS
#####################################
# GET SCAN TR_ID ITERATOR
scan_infosource = Node(util.IdentityInterface(fields=['TR_id']), name='scan_infosource')
scan_infosource.iterables = ('TR_id', TR_list)
# get atlas data
templates_atlases = { # 'GM_mask_MNI_2mm': 'SPM_GM/SPM_GM_mask_2mm.nii.gz',
# 'GM_mask_MNI_3mm': 'SPM_GM/SPM_GM_mask_3mm.nii.gz',
'FSL_MNI_3mm_template': 'MNI152_T1_3mm_brain.nii.gz',
'vmhc_symm_brain': 'cpac_image_resources/symmetric/MNI152_T1_2mm_brain_symmetric.nii.gz',
'vmhc_symm_brain_3mm': 'cpac_image_resources/symmetric/MNI152_T1_3mm_brain_symmetric.nii.gz',
'vmhc_symm_skull': 'cpac_image_resources/symmetric/MNI152_T1_2mm_symmetric.nii.gz',
'vmhc_symm_brain_mask_dil': 'cpac_image_resources/symmetric/MNI152_T1_2mm_brain_mask_symmetric_dil.nii.gz',
'vmhc_config_file_2mm': 'cpac_image_resources/symmetric/T1_2_MNI152_2mm_symmetric.cnf'
}
selectfiles_anat_templates = Node(nio.SelectFiles(templates_atlases,
base_directory=template_dir),
name="selectfiles_anat_templates")
# GET SUBJECT SPECIFIC FUNCTIONAL AND STRUCTURAL DATA
selectfiles_templates = {
'epi_2_MNI_warp': '{subject_id}/rsfMRI_preprocessing/registration/epi_2_MNI_warp/TR_{TR_id}/*.nii.gz',
'epi_mask': '{subject_id}/rsfMRI_preprocessing/masks/brain_mask_epiSpace/TR_{TR_id}/*.nii.gz',
'preproc_epi_full_spectrum': '{subject_id}/rsfMRI_preprocessing/epis/01_denoised/TR_{TR_id}/*.nii.gz',
'preproc_epi_bp': '{subject_id}/rsfMRI_preprocessing/epis/02_denoised_BP/TR_{TR_id}/*.nii.gz',
'preproc_epi_bp_tNorm': '{subject_id}/rsfMRI_preprocessing/epis/03_denoised_BP_tNorm/TR_{TR_id}/*.nii.gz',
'epi_2_struct_mat': '{subject_id}/rsfMRI_preprocessing/registration/epi_2_struct_mat/TR_{TR_id}/*.mat',
't1w': '{subject_id}/raw_niftis/sMRI/t1w_reoriented.nii.gz',
't1w_brain': '{subject_id}/rsfMRI_preprocessing/struct_prep/t1w_brain/t1w_reoriented_maths.nii.gz',
}
selectfiles = Node(nio.SelectFiles(selectfiles_templates,
base_directory=preprocessed_data_dir),
name="selectfiles")
wf.connect(scan_infosource, 'TR_id', selectfiles, 'TR_id')
selectfiles.inputs.subject_id = subject_id
# CREATE TRANSFORMATIONS
# creat MNI 2 epi warp
MNI_2_epi_warp = Node(fsl.InvWarp(), name='MNI_2_epi_warp')
MNI_2_epi_warp.inputs.reference = fsl.Info.standard_image('MNI152_T1_2mm.nii.gz')
wf.connect(selectfiles, 'epi_mask', MNI_2_epi_warp, 'reference')
wf.connect(selectfiles, 'epi_2_MNI_warp', MNI_2_epi_warp, 'warp')
# # CREATE GM MASK IN EPI SPACE
# GM_mask_epiSpace = Node(fsl.ApplyWarp(), name='GM_mask_epiSpace')
# GM_mask_epiSpace.inputs.out_file = 'GM_mask_epiSpace.nii.gz'
#
# wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_2mm', GM_mask_epiSpace, 'in_file')
# wf.connect(selectfiles, 'epi_mask', GM_mask_epiSpace, 'ref_file')
# wf.connect(MNI_2_epi_warp, 'inverse_warp', GM_mask_epiSpace, 'field_file')
# wf.connect(GM_mask_epiSpace, 'out_file', ds, 'GM_mask_epiSpace')
# fixme
# # CREATE TS IN MNI SPACE
# # is it ok to apply the 2mm warpfield to the 3mm template?
# # seems ok: https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind0904&L=FSL&P=R14011&1=FSL&9=A&J=on&d=No+Match%3BMatch%3BMatches&z=4
# epi_bp_MNIspace_3mm = Node(fsl.ApplyWarp(), name='epi_bp_MNIspace_3mm')
# epi_bp_MNIspace_3mm.inputs.interp = 'spline'
# epi_bp_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'}
# wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', epi_bp_MNIspace_3mm, 'ref_file')
# wf.connect(selectfiles, 'preproc_epi_bp', epi_bp_MNIspace_3mm, 'in_file')
# wf.connect(selectfiles, 'epi_2_MNI_warp', epi_bp_MNIspace_3mm, 'field_file')
# CREATE EPI MASK IN MNI SPACE
epi_mask_MNIspace_3mm = Node(fsl.ApplyWarp(), name='epi_mask_MNIspace_3mm')
epi_mask_MNIspace_3mm.inputs.interp = 'nn'
epi_mask_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'}
wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', epi_mask_MNIspace_3mm, 'ref_file')
wf.connect(selectfiles, 'epi_mask', epi_mask_MNIspace_3mm, 'in_file')
wf.connect(selectfiles, 'epi_2_MNI_warp', epi_mask_MNIspace_3mm, 'field_file')
wf.connect(epi_mask_MNIspace_3mm, 'out_file', ds, 'epi_mask_MNIspace_3mm')
#####################
# CALCULATE METRICS
#####################
# f/ALFF
alff = cpac_alff.create_alff('alff')
alff.inputs.hp_input.hp = 0.01
alff.inputs.lp_input.lp = 0.1
wf.connect(selectfiles, 'preproc_epi_full_spectrum', alff, 'inputspec.rest_res')
# wf.connect(GM_mask_epiSpace, 'out_file', alff, 'inputspec.rest_mask')
wf.connect(selectfiles, 'epi_mask', alff, 'inputspec.rest_mask')
wf.connect(alff, 'outputspec.alff_img', ds, 'alff.alff')
wf.connect(alff, 'outputspec.falff_img', ds, 'alff.falff')
# f/ALFF 2 MNI
# fixme spline or default?
alff_MNIspace_3mm = Node(fsl.ApplyWarp(), name='alff_MNIspace_3mm')
alff_MNIspace_3mm.inputs.interp = 'spline'
alff_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'}
wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', alff_MNIspace_3mm, 'ref_file')
wf.connect(alff, 'outputspec.alff_img', alff_MNIspace_3mm, 'in_file')
wf.connect(selectfiles, 'epi_2_MNI_warp', alff_MNIspace_3mm, 'field_file')
wf.connect(alff_MNIspace_3mm, 'out_file', ds, 'alff.alff_MNI_3mm')
falff_MNIspace_3mm = Node(fsl.ApplyWarp(), name='falff_MNIspace_3mm')
falff_MNIspace_3mm.inputs.interp = 'spline'
falff_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'}
wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', falff_MNIspace_3mm, 'ref_file')
wf.connect(alff, 'outputspec.falff_img', falff_MNIspace_3mm, 'in_file')
wf.connect(selectfiles, 'epi_2_MNI_warp', falff_MNIspace_3mm, 'field_file')
wf.connect(falff_MNIspace_3mm, 'out_file', ds, 'alff.falff_MNI_3mm')
# f/ALFF_MNI Z-SCORE
alff_MNIspace_3mm_Z = cpac_utils.get_zscore(input_name='alff_MNIspace_3mm', wf_name='alff_MNIspace_3mm_Z')
wf.connect(alff_MNIspace_3mm, 'out_file', alff_MNIspace_3mm_Z, 'inputspec.input_file')
# wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', alff_MNIspace_3mm_Z, 'inputspec.mask_file')
wf.connect(epi_mask_MNIspace_3mm, 'out_file', alff_MNIspace_3mm_Z, 'inputspec.mask_file')
wf.connect(alff_MNIspace_3mm_Z, 'outputspec.z_score_img', ds, 'alff.alff_MNI_3mm_Z')
falff_MNIspace_3mm_Z = cpac_utils.get_zscore(input_name='falff_MNIspace_3mm', wf_name='falff_MNIspace_3mm_Z')
wf.connect(falff_MNIspace_3mm, 'out_file', falff_MNIspace_3mm_Z, 'inputspec.input_file')
# wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', falff_MNIspace_3mm_Z, 'inputspec.mask_file')
wf.connect(epi_mask_MNIspace_3mm, 'out_file', falff_MNIspace_3mm_Z, 'inputspec.mask_file')
wf.connect(falff_MNIspace_3mm_Z, 'outputspec.z_score_img', ds, 'alff.falff_MNI_3mm_Z')
# f/ALFF_MNI STANDARDIZE BY MEAN
alff_MNIspace_3mm_standardized_mean = calc_metrics_utils.standardize_divide_by_mean(
wf_name='alff_MNIspace_3mm_standardized_mean')
wf.connect(alff_MNIspace_3mm, 'out_file', alff_MNIspace_3mm_standardized_mean, 'inputnode.in_file')
wf.connect(epi_mask_MNIspace_3mm, 'out_file', alff_MNIspace_3mm_standardized_mean, 'inputnode.mask_file')
wf.connect(alff_MNIspace_3mm_standardized_mean, 'outputnode.out_file', ds, 'alff.alff_MNI_3mm_standardized_mean')
falff_MNIspace_3mm_standardized_mean = calc_metrics_utils.standardize_divide_by_mean(
wf_name='falff_MNIspace_3mm_standardized_mean')
wf.connect(falff_MNIspace_3mm, 'out_file', falff_MNIspace_3mm_standardized_mean, 'inputnode.in_file')
wf.connect(epi_mask_MNIspace_3mm, 'out_file', falff_MNIspace_3mm_standardized_mean, 'inputnode.mask_file')
wf.connect(falff_MNIspace_3mm_standardized_mean, 'outputnode.out_file', ds, 'alff.falff_MNI_3mm_standardized_mean')
# REHO
reho = cpac_reho.create_reho()
reho.inputs.inputspec.cluster_size = 27
wf.connect(selectfiles, 'preproc_epi_bp', reho, 'inputspec.rest_res_filt')
# wf.connect(GM_mask_epiSpace, 'out_file', reho, 'inputspec.rest_mask')
wf.connect(selectfiles, 'epi_mask', reho, 'inputspec.rest_mask')
wf.connect(reho, 'outputspec.raw_reho_map', ds, 'reho.reho')
# REHO 2 MNI
# fixme spline or default?
reho_MNIspace_3mm = Node(fsl.ApplyWarp(), name='reho_MNIspace_3mm')
reho_MNIspace_3mm.inputs.interp = 'spline'
reho_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'}
wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', reho_MNIspace_3mm, 'ref_file')
wf.connect(reho, 'outputspec.raw_reho_map', reho_MNIspace_3mm, 'in_file')
wf.connect(selectfiles, 'epi_2_MNI_warp', reho_MNIspace_3mm, 'field_file')
wf.connect(reho_MNIspace_3mm, 'out_file', ds, 'reho.reho_MNI_3mm')
# REHO_MNI Z-SCORE
reho_MNIspace_3mm_Z = cpac_utils.get_zscore(input_name='reho_MNIspace_3mm', wf_name='reho_MNIspace_3mm_Z')
wf.connect(alff_MNIspace_3mm, 'out_file', reho_MNIspace_3mm_Z, 'inputspec.input_file')
# wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', reho_MNIspace_3mm_Z, 'inputspec.mask_file')
wf.connect(epi_mask_MNIspace_3mm, 'out_file', reho_MNIspace_3mm_Z, 'inputspec.mask_file')
wf.connect(reho_MNIspace_3mm_Z, 'outputspec.z_score_img', ds, 'reho.reho_MNI_3mm_Z')
# REHO_MNI STANDARDIZE BY MEAN
reho_MNIspace_3mm_standardized_mean = calc_metrics_utils.standardize_divide_by_mean(
wf_name='reho_MNIspace_3mm_standardized_mean')
wf.connect(reho_MNIspace_3mm, 'out_file', reho_MNIspace_3mm_standardized_mean, 'inputnode.in_file')
wf.connect(epi_mask_MNIspace_3mm, 'out_file', reho_MNIspace_3mm_standardized_mean, 'inputnode.mask_file')
wf.connect(reho_MNIspace_3mm_standardized_mean, 'outputnode.out_file', ds, 'reho.reho_MNI_3mm_standardized_mean')
# VMHC
# create registration to symmetrical MNI template
struct_2_MNI_symm = cpac_registration.create_nonlinear_register(name='struct_2_MNI_symm')
wf.connect(selectfiles_anat_templates, 'vmhc_config_file_2mm', struct_2_MNI_symm, 'inputspec.fnirt_config')
wf.connect(selectfiles_anat_templates, 'vmhc_symm_brain', struct_2_MNI_symm, 'inputspec.reference_brain')
wf.connect(selectfiles_anat_templates, 'vmhc_symm_skull', struct_2_MNI_symm, 'inputspec.reference_skull')
wf.connect(selectfiles_anat_templates, 'vmhc_symm_brain_mask_dil', struct_2_MNI_symm, 'inputspec.ref_mask')
wf.connect(selectfiles, 't1w', struct_2_MNI_symm, 'inputspec.input_skull')
wf.connect(selectfiles, 't1w_brain', struct_2_MNI_symm, 'inputspec.input_brain')
wf.connect(struct_2_MNI_symm, 'outputspec.output_brain', ds, 'vmhc.symm_reg.@output_brain')
wf.connect(struct_2_MNI_symm, 'outputspec.linear_xfm', ds, 'vmhc.symm_reg.@linear_xfm')
wf.connect(struct_2_MNI_symm, 'outputspec.invlinear_xfm', ds, 'vmhc.symm_reg.@invlinear_xfm')
wf.connect(struct_2_MNI_symm, 'outputspec.nonlinear_xfm', ds, 'vmhc.symm_reg.@nonlinear_xfm')
# fixme
vmhc = cpac_vmhc.create_vmhc(use_ants=False, name='vmhc')
vmhc.inputs.fwhm_input.fwhm = 4
wf.connect(selectfiles_anat_templates, 'vmhc_symm_brain_3mm', vmhc, 'inputspec.standard_for_func')
wf.connect(selectfiles, 'preproc_epi_bp_tNorm', vmhc, 'inputspec.rest_res')
wf.connect(selectfiles, 'epi_2_struct_mat', vmhc, 'inputspec.example_func2highres_mat')
wf.connect(struct_2_MNI_symm, 'outputspec.nonlinear_xfm', vmhc, 'inputspec.fnirt_nonlinear_warp')
# wf.connect(GM_mask_epiSpace, 'out_file', vmhc, 'inputspec.rest_mask')
wf.connect(selectfiles, 'epi_mask', vmhc, 'inputspec.rest_mask')
wf.connect(vmhc, 'outputspec.rest_res_2symmstandard', ds, 'vmhc.rest_res_2symmstandard')
wf.connect(vmhc, 'outputspec.VMHC_FWHM_img', ds, 'vmhc.VMHC_FWHM_img')
wf.connect(vmhc, 'outputspec.VMHC_Z_FWHM_img', ds, 'vmhc.VMHC_Z_FWHM_img')
wf.connect(vmhc, 'outputspec.VMHC_Z_stat_FWHM_img', ds, 'vmhc.VMHC_Z_stat_FWHM_img')
# VARIABILITY SCORES
variability = Node(util.Function(input_names=['in_file'],
output_names=['out_file_list'],
function=calc_metrics_utils.calc_variability),
name='variability')
wf.connect(selectfiles, 'preproc_epi_bp', variability, 'in_file')
wf.connect(variability, 'out_file_list', ds, 'variability.subjectSpace.@out_files')
# #fixme spline?
variabilty_MNIspace_3mm = MapNode(fsl.ApplyWarp(), iterfield=['in_file'], name='variabilty_MNIspace_3mm')
variabilty_MNIspace_3mm.inputs.interp = 'spline'
variabilty_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'}
wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', variabilty_MNIspace_3mm, 'ref_file')
wf.connect(selectfiles, 'epi_2_MNI_warp', variabilty_MNIspace_3mm, 'field_file')
wf.connect(variability, 'out_file_list', variabilty_MNIspace_3mm, 'in_file')
wf.connect(variabilty_MNIspace_3mm, 'out_file', ds, 'variability.MNI_3mm.@out_file')
# CALC Z SCORE
variabilty_MNIspace_3mm_Z = cpac_centrality_z_score.get_cent_zscore(wf_name='variabilty_MNIspace_3mm_Z')
wf.connect(variabilty_MNIspace_3mm, 'out_file', variabilty_MNIspace_3mm_Z, 'inputspec.input_file')
# wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', variabilty_MNIspace_3mm_Z, 'inputspec.mask_file')
wf.connect(epi_mask_MNIspace_3mm, 'out_file', variabilty_MNIspace_3mm_Z, 'inputspec.mask_file')
wf.connect(variabilty_MNIspace_3mm_Z, 'outputspec.z_score_img', ds, 'variability.MNI_3mm_Z.@out_file')
# STANDARDIZE BY MEAN
variabilty_MNIspace_3mm_standardized_mean = calc_metrics_utils.standardize_divide_by_mean(
wf_name='variabilty_MNIspace_3mm_standardized_mean')
wf.connect(variabilty_MNIspace_3mm, 'out_file', variabilty_MNIspace_3mm_standardized_mean, 'inputnode.in_file')
wf.connect(epi_mask_MNIspace_3mm, 'out_file', variabilty_MNIspace_3mm_standardized_mean, 'inputnode.mask_file')
wf.connect(variabilty_MNIspace_3mm_standardized_mean, 'outputnode.out_file', ds,
'variability.MNI_3mm_standardized_mean.@out_file')
wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical')
wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf')
wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf')
if plugin_name == 'CondorDAGMan':
wf.run(plugin=plugin_name)
if plugin_name == 'MultiProc':
wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def segmentation(projectid, subjectid, sessionid, master_config, onlyT1=True, pipeline_name=''):
import os.path
import nipype.pipeline.engine as pe
import nipype.interfaces.io as nio
from nipype.interfaces import ants
from nipype.interfaces.utility import IdentityInterface, Function, Merge
# Set universal pipeline options
from nipype import config
config.update_config(master_config)
from PipeLineFunctionHelpers import ClipT1ImageWithBrainMask
from .WorkupT1T2BRAINSCut import CreateBRAINSCutWorkflow
from utilities.distributed import modify_qsub_args
from nipype.interfaces.semtools import BRAINSSnapShotWriter
# CLUSTER_QUEUE=master_config['queue']
CLUSTER_QUEUE_LONG = master_config['long_q']
baw200 = pe.Workflow(name=pipeline_name)
# HACK: print for debugging
for key, itme in list(master_config.items()):
print(("-" * 30))
print((key, ":", itme))
print(("-" * 30))
# END HACK
inputsSpec = pe.Node(interface=IdentityInterface(fields=['t1_average',
't2_average',
'template_t1',
'hncma_atlas',
'LMIatlasToSubject_tx',
'inputLabels',
'inputHeadLabels',
'posteriorImages',
'UpdatedPosteriorsList',
'atlasToSubjectRegistrationState',
'rho',
'phi',
'theta',
'l_caudate_ProbabilityMap',
'r_caudate_ProbabilityMap',
'l_hippocampus_ProbabilityMap',
'r_hippocampus_ProbabilityMap',
'l_putamen_ProbabilityMap',
'r_putamen_ProbabilityMap',
'l_thalamus_ProbabilityMap',
'r_thalamus_ProbabilityMap',
'l_accumben_ProbabilityMap',
'r_accumben_ProbabilityMap',
'l_globus_ProbabilityMap',
'r_globus_ProbabilityMap',
'trainModelFile_txtD0060NT0060_gz',
]),
run_without_submitting=True, name='inputspec')
# outputsSpec = pe.Node(interface=IdentityInterface(fields=[...]),
# run_without_submitting=True, name='outputspec')
currentClipT1ImageWithBrainMaskName = 'ClipT1ImageWithBrainMask_' + str(subjectid) + "_" + str(sessionid)
ClipT1ImageWithBrainMaskNode = pe.Node(interface=Function(function=ClipT1ImageWithBrainMask,
input_names=['t1_image', 'brain_labels',
'clipped_file_name'],
output_names=['clipped_file']),
name=currentClipT1ImageWithBrainMaskName)
ClipT1ImageWithBrainMaskNode.inputs.clipped_file_name = 'clipped_from_BABC_labels_t1.nii.gz'
baw200.connect([(inputsSpec, ClipT1ImageWithBrainMaskNode, [('t1_average', 't1_image'),
('inputLabels', 'brain_labels')])])
currentA2SantsRegistrationPostABCSyN = 'A2SantsRegistrationPostABCSyN_' + str(subjectid) + "_" + str(sessionid)
## TODO: It would be great to update the BRAINSABC atlasToSubjectTransform at this point, but
## That requires more testing, and fixes to ANTS to properly collapse transforms.
## For now we are simply creating a dummy node to pass through
A2SantsRegistrationPostABCSyN = pe.Node(interface=ants.Registration(), name=currentA2SantsRegistrationPostABCSyN)
many_cpu_ANTsSyN_options_dictionary = {'qsub_args': modify_qsub_args(CLUSTER_QUEUE_LONG, 8, 8, 16),
'overwrite': True}
A2SantsRegistrationPostABCSyN.plugin_args = many_cpu_ANTsSyN_options_dictionary
CommonANTsRegistrationSettings(
antsRegistrationNode=A2SantsRegistrationPostABCSyN,
registrationTypeDescription="A2SantsRegistrationPostABCSyN",
output_transform_prefix='AtlasToSubjectPostBABC_SyN',
output_warped_image='atlas2subjectPostBABC.nii.gz',
output_inverse_warped_image='subject2atlasPostBABC.nii.gz',
save_state='SavedInternalSyNStatePostBABC.h5',
invert_initial_moving_transform=False,
initial_moving_transform=None)
## TODO: Try multi-modal registration here
baw200.connect([(inputsSpec, A2SantsRegistrationPostABCSyN, [('atlasToSubjectRegistrationState', 'restore_state'),
('t1_average', 'fixed_image'),
('template_t1', 'moving_image')])
])
myLocalSegWF = CreateBRAINSCutWorkflow(projectid,
subjectid,
sessionid,
master_config['queue'],
master_config['long_q'],
"Segmentation",
onlyT1)
MergeStage2AverageImagesName = "99_mergeAvergeStage2Images_" + str(sessionid)
MergeStage2AverageImages = pe.Node(interface=Merge(2), run_without_submitting=True,
name=MergeStage2AverageImagesName)
baw200.connect([(inputsSpec, myLocalSegWF, [('t1_average', 'inputspec.T1Volume'),
('template_t1', 'inputspec.template_t1'),
('posteriorImages', "inputspec.posteriorDictionary"),
('inputLabels', 'inputspec.RegistrationROI'), ]),
(inputsSpec, MergeStage2AverageImages, [('t1_average', 'in1')]),
(A2SantsRegistrationPostABCSyN, myLocalSegWF, [('composite_transform',
'inputspec.atlasToSubjectTransform')])
])
baw200.connect([(inputsSpec, myLocalSegWF,
[
('rho', 'inputspec.rho'),
('phi', 'inputspec.phi'),
('theta', 'inputspec.theta'),
('l_caudate_ProbabilityMap', 'inputspec.l_caudate_ProbabilityMap'),
('r_caudate_ProbabilityMap', 'inputspec.r_caudate_ProbabilityMap'),
('l_hippocampus_ProbabilityMap', 'inputspec.l_hippocampus_ProbabilityMap'),
('r_hippocampus_ProbabilityMap', 'inputspec.r_hippocampus_ProbabilityMap'),
('l_putamen_ProbabilityMap', 'inputspec.l_putamen_ProbabilityMap'),
('r_putamen_ProbabilityMap', 'inputspec.r_putamen_ProbabilityMap'),
('l_thalamus_ProbabilityMap', 'inputspec.l_thalamus_ProbabilityMap'),
('r_thalamus_ProbabilityMap', 'inputspec.r_thalamus_ProbabilityMap'),
('l_accumben_ProbabilityMap', 'inputspec.l_accumben_ProbabilityMap'),
('r_accumben_ProbabilityMap', 'inputspec.r_accumben_ProbabilityMap'),
('l_globus_ProbabilityMap', 'inputspec.l_globus_ProbabilityMap'),
('r_globus_ProbabilityMap', 'inputspec.r_globus_ProbabilityMap'),
('trainModelFile_txtD0060NT0060_gz', 'inputspec.trainModelFile_txtD0060NT0060_gz')
]
)]
)
if not onlyT1:
baw200.connect([(inputsSpec, myLocalSegWF, [('t2_average', 'inputspec.T2Volume')]),
(inputsSpec, MergeStage2AverageImages, [('t2_average', 'in2')])])
file_count = 15 # Count of files to merge into MergeSessionSubjectToAtlas
else:
file_count = 14 # Count of files to merge into MergeSessionSubjectToAtlas
## NOTE: Element 0 of AccumulatePriorsList is the accumulated GM tissue
# baw200.connect([(AccumulateLikeTissuePosteriorsNode, myLocalSegWF,
# [(('AccumulatePriorsList', getListIndex, 0), "inputspec.TotalGM")]),
# ])
### Now define where the final organized outputs should go.
DataSink = pe.Node(nio.DataSink(), name="CleanedDenoisedSegmentation_DS_" + str(subjectid) + "_" + str(sessionid))
DataSink.overwrite = master_config['ds_overwrite']
DataSink.inputs.base_directory = master_config['resultdir']
# DataSink.inputs.regexp_substitutions = GenerateOutputPattern(projectid, subjectid, sessionid,'BRAINSCut')
# DataSink.inputs.regexp_substitutions = GenerateBRAINSCutImagesOutputPattern(projectid, subjectid, sessionid)
DataSink.inputs.substitutions = [
('Segmentations', os.path.join(projectid, subjectid, sessionid, 'CleanedDenoisedRFSegmentations')),
('subjectANNLabel_', ''),
('ANNContinuousPrediction', ''),
('subject.nii.gz', '.nii.gz'),
('_seg.nii.gz', '_seg.nii.gz'),
('.nii.gz', '_seg.nii.gz'),
('_seg_seg', '_seg')]
baw200.connect([(myLocalSegWF, DataSink, [('outputspec.outputBinaryLeftCaudate', 'Segmentations.@LeftCaudate'),
('outputspec.outputBinaryRightCaudate', 'Segmentations.@RightCaudate'),
('outputspec.outputBinaryLeftHippocampus',
'Segmentations.@LeftHippocampus'),
('outputspec.outputBinaryRightHippocampus',
'Segmentations.@RightHippocampus'),
('outputspec.outputBinaryLeftPutamen', 'Segmentations.@LeftPutamen'),
('outputspec.outputBinaryRightPutamen', 'Segmentations.@RightPutamen'),
('outputspec.outputBinaryLeftThalamus', 'Segmentations.@LeftThalamus'),
('outputspec.outputBinaryRightThalamus', 'Segmentations.@RightThalamus'),
('outputspec.outputBinaryLeftAccumben', 'Segmentations.@LeftAccumben'),
('outputspec.outputBinaryRightAccumben', 'Segmentations.@RightAccumben'),
('outputspec.outputBinaryLeftGlobus', 'Segmentations.@LeftGlobus'),
('outputspec.outputBinaryRightGlobus', 'Segmentations.@RightGlobus'),
('outputspec.outputLabelImageName', 'Segmentations.@LabelImageName'),
('outputspec.outputCSVFileName', 'Segmentations.@CSVFileName')]),
# (myLocalSegWF, DataSink, [('outputspec.cleaned_labels', 'Segmentations.@cleaned_labels')])
])
MergeStage2BinaryVolumesName = "99_MergeStage2BinaryVolumes_" + str(sessionid)
MergeStage2BinaryVolumes = pe.Node(interface=Merge(12), run_without_submitting=True,
name=MergeStage2BinaryVolumesName)
baw200.connect([(myLocalSegWF, MergeStage2BinaryVolumes, [('outputspec.outputBinaryLeftAccumben', 'in1'),
('outputspec.outputBinaryLeftCaudate', 'in2'),
('outputspec.outputBinaryLeftPutamen', 'in3'),
('outputspec.outputBinaryLeftGlobus', 'in4'),
('outputspec.outputBinaryLeftThalamus', 'in5'),
('outputspec.outputBinaryLeftHippocampus', 'in6'),
('outputspec.outputBinaryRightAccumben', 'in7'),
('outputspec.outputBinaryRightCaudate', 'in8'),
('outputspec.outputBinaryRightPutamen', 'in9'),
('outputspec.outputBinaryRightGlobus', 'in10'),
('outputspec.outputBinaryRightThalamus', 'in11'),
('outputspec.outputBinaryRightHippocampus', 'in12')])
])
## SnapShotWriter for Segmented result checking:
SnapShotWriterNodeName = "SnapShotWriter_" + str(sessionid)
SnapShotWriter = pe.Node(interface=BRAINSSnapShotWriter(), name=SnapShotWriterNodeName)
SnapShotWriter.inputs.outputFilename = 'snapShot' + str(sessionid) + '.png' # output specification
SnapShotWriter.inputs.inputPlaneDirection = [2, 1, 1, 1, 1, 0, 0]
SnapShotWriter.inputs.inputSliceToExtractInPhysicalPoint = [-3, -7, -3, 5, 7, 22, -22]
baw200.connect([(MergeStage2AverageImages, SnapShotWriter, [('out', 'inputVolumes')]),
(MergeStage2BinaryVolumes, SnapShotWriter, [('out', 'inputBinaryVolumes')]),
(SnapShotWriter, DataSink, [('outputFilename', 'Segmentations.@outputSnapShot')])
])
# currentAntsLabelWarpToSubject = 'AntsLabelWarpToSubject' + str(subjectid) + "_" + str(sessionid)
# AntsLabelWarpToSubject = pe.Node(interface=ants.ApplyTransforms(), name=currentAntsLabelWarpToSubject)
#
# AntsLabelWarpToSubject.inputs.num_threads = -1
# AntsLabelWarpToSubject.inputs.dimension = 3
# AntsLabelWarpToSubject.inputs.output_image = 'warped_hncma_atlas_seg.nii.gz'
# AntsLabelWarpToSubject.inputs.interpolation = "MultiLabel"
#
# baw200.connect([(A2SantsRegistrationPostABCSyN, AntsLabelWarpToSubject, [('composite_transform', 'transforms')]),
# (inputsSpec, AntsLabelWarpToSubject, [('t1_average', 'reference_image'),
# ('hncma_atlas', 'input_image')])
# ])
# #####
# ### Now define where the final organized outputs should go.
# AntsLabelWarpedToSubject_DSName = "AntsLabelWarpedToSubject_DS_" + str(sessionid)
# AntsLabelWarpedToSubject_DS = pe.Node(nio.DataSink(), name=AntsLabelWarpedToSubject_DSName)
# AntsLabelWarpedToSubject_DS.overwrite = master_config['ds_overwrite']
# AntsLabelWarpedToSubject_DS.inputs.base_directory = master_config['resultdir']
# AntsLabelWarpedToSubject_DS.inputs.substitutions = [('AntsLabelWarpedToSubject', os.path.join(projectid, subjectid, sessionid, 'AntsLabelWarpedToSubject'))]
#
# baw200.connect([(AntsLabelWarpToSubject, AntsLabelWarpedToSubject_DS, [('output_image', 'AntsLabelWarpedToSubject')])])
MergeSessionSubjectToAtlasName = "99_MergeSessionSubjectToAtlas_" + str(sessionid)
MergeSessionSubjectToAtlas = pe.Node(interface=Merge(file_count), run_without_submitting=True,
name=MergeSessionSubjectToAtlasName)
baw200.connect([(myLocalSegWF, MergeSessionSubjectToAtlas, [('outputspec.outputBinaryLeftAccumben', 'in1'),
('outputspec.outputBinaryLeftCaudate', 'in2'),
('outputspec.outputBinaryLeftPutamen', 'in3'),
('outputspec.outputBinaryLeftGlobus', 'in4'),
('outputspec.outputBinaryLeftThalamus', 'in5'),
('outputspec.outputBinaryLeftHippocampus', 'in6'),
('outputspec.outputBinaryRightAccumben', 'in7'),
('outputspec.outputBinaryRightCaudate', 'in8'),
('outputspec.outputBinaryRightPutamen', 'in9'),
('outputspec.outputBinaryRightGlobus', 'in10'),
('outputspec.outputBinaryRightThalamus', 'in11'),
('outputspec.outputBinaryRightHippocampus', 'in12')]),
# (FixWMPartitioningNode, MergeSessionSubjectToAtlas, [('UpdatedPosteriorsList', 'in13')]),
(inputsSpec, MergeSessionSubjectToAtlas, [('UpdatedPosteriorsList', 'in13')]),
(inputsSpec, MergeSessionSubjectToAtlas, [('t1_average', 'in14')])
])
if not onlyT1:
assert file_count == 15
baw200.connect([(inputsSpec, MergeSessionSubjectToAtlas, [('t2_average', 'in15')])])
LinearSubjectToAtlasANTsApplyTransformsName = 'LinearSubjectToAtlasANTsApplyTransforms_' + str(sessionid)
LinearSubjectToAtlasANTsApplyTransforms = pe.MapNode(interface=ants.ApplyTransforms(), iterfield=['input_image'],
name=LinearSubjectToAtlasANTsApplyTransformsName)
LinearSubjectToAtlasANTsApplyTransforms.inputs.num_threads = -1
LinearSubjectToAtlasANTsApplyTransforms.inputs.interpolation = 'Linear'
baw200.connect(
[(A2SantsRegistrationPostABCSyN, LinearSubjectToAtlasANTsApplyTransforms, [('inverse_composite_transform',
'transforms')]),
(inputsSpec, LinearSubjectToAtlasANTsApplyTransforms, [('template_t1', 'reference_image')]),
(MergeSessionSubjectToAtlas, LinearSubjectToAtlasANTsApplyTransforms, [('out', 'input_image')])
])
MergeMultiLabelSessionSubjectToAtlasName = "99_MergeMultiLabelSessionSubjectToAtlas_" + str(sessionid)
MergeMultiLabelSessionSubjectToAtlas = pe.Node(interface=Merge(2), run_without_submitting=True,
name=MergeMultiLabelSessionSubjectToAtlasName)
baw200.connect([(inputsSpec, MergeMultiLabelSessionSubjectToAtlas, [('inputLabels', 'in1'),
('inputHeadLabels', 'in2')])
])
### This is taking this sessions RF label map back into NAC atlas space.
# {
MultiLabelSubjectToAtlasANTsApplyTransformsName = 'MultiLabelSubjectToAtlasANTsApplyTransforms_' + str(
sessionid) + '_map'
MultiLabelSubjectToAtlasANTsApplyTransforms = pe.MapNode(interface=ants.ApplyTransforms(),
iterfield=['input_image'],
name=MultiLabelSubjectToAtlasANTsApplyTransformsName)
MultiLabelSubjectToAtlasANTsApplyTransforms.inputs.num_threads = -1
MultiLabelSubjectToAtlasANTsApplyTransforms.inputs.interpolation = 'MultiLabel'
baw200.connect([(A2SantsRegistrationPostABCSyN, MultiLabelSubjectToAtlasANTsApplyTransforms,
[('inverse_composite_transform', 'transforms')]),
(inputsSpec, MultiLabelSubjectToAtlasANTsApplyTransforms, [('template_t1', 'reference_image')]),
(MergeMultiLabelSessionSubjectToAtlas, MultiLabelSubjectToAtlasANTsApplyTransforms,
[('out', 'input_image')])
])
# }
### Now we must take the sessions to THIS SUBJECTS personalized atlas.
# {
# }
### Now define where the final organized outputs should go.
Subj2Atlas_DSName = "SubjectToAtlas_DS_" + str(sessionid)
Subj2Atlas_DS = pe.Node(nio.DataSink(), name=Subj2Atlas_DSName)
Subj2Atlas_DS.overwrite = master_config['ds_overwrite']
Subj2Atlas_DS.inputs.base_directory = master_config['resultdir']
Subj2Atlas_DS.inputs.regexp_substitutions = [(r'_LinearSubjectToAtlasANTsApplyTransforms_[^/]*',
r'' + sessionid + '/')]
baw200.connect([(LinearSubjectToAtlasANTsApplyTransforms, Subj2Atlas_DS,
[('output_image', 'SubjectToAtlasWarped.@linear_output_images')])])
Subj2AtlasTransforms_DSName = "SubjectToAtlasTransforms_DS_" + str(sessionid)
Subj2AtlasTransforms_DS = pe.Node(nio.DataSink(), name=Subj2AtlasTransforms_DSName)
Subj2AtlasTransforms_DS.overwrite = master_config['ds_overwrite']
Subj2AtlasTransforms_DS.inputs.base_directory = master_config['resultdir']
Subj2AtlasTransforms_DS.inputs.regexp_substitutions = [(r'SubjectToAtlasWarped',
r'SubjectToAtlasWarped/' + sessionid + '/')]
baw200.connect([(A2SantsRegistrationPostABCSyN, Subj2AtlasTransforms_DS,
[('composite_transform', 'SubjectToAtlasWarped.@composite_transform'),
('inverse_composite_transform', 'SubjectToAtlasWarped.@inverse_composite_transform')])])
# baw200.connect([(MultiLabelSubjectToAtlasANTsApplyTransforms, Subj2Atlas_DS, [('output_image', 'SubjectToAtlasWarped.@multilabel_output_images')])])
if master_config['plugin_name'].startswith(
'SGE'): # for some nodes, the qsub call needs to be modified on the cluster
A2SantsRegistrationPostABCSyN.plugin_args = {'template': master_config['plugin_args']['template'],
'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'], 8, 8, 24)}
SnapShotWriter.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'], 1, 1, 1)}
LinearSubjectToAtlasANTsApplyTransforms.plugin_args = {'template': master_config['plugin_args']['template'],
'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'], 1,
1, 1)}
MultiLabelSubjectToAtlasANTsApplyTransforms.plugin_args = {'template': master_config['plugin_args']['template'],
'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'],
1, 1, 1)}
return baw200
def main():
"""Entry point"""
parser = ArgumentParser(description='MRI Quality Control',
formatter_class=RawTextHelpFormatter)
g_input = parser.add_argument_group('Inputs')
g_input.add_argument('-B', '--bids-root', action='store', default=os.getcwd())
g_input.add_argument('-i', '--input-folder', action='store')
g_input.add_argument('-S', '--subject-id', nargs='*', action='store')
g_input.add_argument('-s', '--session-id', action='store')
g_input.add_argument('-r', '--run-id', action='store')
g_input.add_argument('-d', '--data-type', action='store', nargs='*',
choices=['anat', 'func'], default=['anat', 'func'])
g_input.add_argument('-v', '--version', action='store_true', default=False,
help='Show current mriqc version')
g_input.add_argument('--nthreads', action='store', default=0,
type=int, help='number of threads')
g_input.add_argument('--write-graph', action='store_true', default=False,
help='Write workflow graph.')
g_input.add_argument('--test-run', action='store_true', default=False,
help='Do not run the workflow.')
g_input.add_argument('--use-plugin', action='store', default=None,
help='nipype plugin configuration file')
g_input.add_argument('--save-memory', action='store_true', default=False,
help='Save as much memory as possible')
g_input.add_argument('--hmc-afni', action='store_true', default=False,
help='Use ANFI 3dvolreg for head motion correction (HMC) and '
'frame displacement (FD) estimation')
g_input.add_argument('--ants-settings', action='store',
help='path to JSON file with settings for ANTS')
g_outputs = parser.add_argument_group('Outputs')
g_outputs.add_argument('-o', '--output-dir', action='store')
g_outputs.add_argument('-w', '--work-dir', action='store', default=op.join(os.getcwd(), 'work'))
opts = parser.parse_args()
bids_root = op.abspath(opts.bids_root)
if opts.input_folder is not None:
warn('The --input-folder flag is deprecated, please use -B instead', DeprecationWarning)
if bids_root == os.getcwd():
bids_root = op.abspath(opts.input_folder)
if opts.version:
print('mriqc version ' + __version__)
exit(0)
settings = {'bids_root': bids_root,
'output_dir': os.getcwd(),
'write_graph': opts.write_graph,
'save_memory': opts.save_memory,
'hmc_afni': opts.hmc_afni,
'nthreads': opts.nthreads}
if opts.output_dir:
settings['output_dir'] = op.abspath(opts.output_dir)
if not op.exists(settings['output_dir']):
os.makedirs(settings['output_dir'])
settings['work_dir'] = op.abspath(opts.work_dir)
with LockFile(settings['work_dir']):
if not op.exists(settings['work_dir']):
os.makedirs(settings['work_dir'])
if opts.ants_settings:
settings['ants_settings'] = opts.ants_settings
log_dir = op.join(settings['work_dir'] + '_log')
if not op.exists(log_dir):
os.makedirs(log_dir)
# Set nipype config
ncfg.update_config({
'logging': {'log_directory': log_dir, 'log_to_file': True},
'execution': {'crashdump_dir': log_dir}
})
plugin_settings = {'plugin': 'Linear'}
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as pfile:
plugin_settings = loadyml(pfile)
else:
# Setup multiprocessing
if settings['nthreads'] == 0:
settings['nthreads'] = cpu_count()
if settings['nthreads'] > 1:
plugin_settings['plugin'] = 'MultiProc'
plugin_settings['plugin_args'] = {'n_procs': settings['nthreads']}
for dtype in opts.data_type:
ms_func = getattr(mwc, 'ms_' + dtype)
workflow = ms_func(subject_id=opts.subject_id, session_id=opts.session_id,
run_id=opts.run_id, settings=settings)
if workflow is None:
LOGGER.warn('No {} scans were found in {}', dtype, settings['bids_root'])
continue
workflow.base_dir = settings['work_dir']
if settings.get('write_graph', False):
workflow.write_graph()
if not opts.test_run:
workflow.run(**plugin_settings)
if opts.subject_id is None and not opts.test_run:
workflow_report(dtype, settings)
def learning_prepare_data_wf(working_dir,
ds_dir,
template_dir,
df_file,
in_data_name_list,
data_lookup_dict,
use_n_procs,
plugin_name):
import os
from nipype import config
from nipype.pipeline.engine import Node, Workflow, MapNode, JoinNode
import nipype.interfaces.utility as util
import nipype.interfaces.io as nio
from nipype.interfaces.freesurfer.utils import ImageInfo
from utils import aggregate_data, vectorize_data
from itertools import chain
# ensure in_data_name_list is list of lists
in_data_name_list = [i if type(i) == list else [i] for i in in_data_name_list]
in_data_name_list_unique = list(set(chain.from_iterable(in_data_name_list)))
#####################################
# GENERAL SETTINGS
#####################################
wf = Workflow(name='learning_prepare_data_wf')
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True,
'remove_unnecessary_outputs': False,
'job_finished_timeout': 120})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash')
ds = Node(nio.DataSink(), name='ds')
ds.inputs.base_directory = os.path.join(ds_dir, 'group_learning_prepare_data')
ds.inputs.regexp_substitutions = [
# ('subject_id_', ''),
('_parcellation_', ''),
('_bp_freqs_', 'bp_'),
('_extraction_method_', ''),
('_subject_id_[A0-9]*/', '')
]
ds_pdf = Node(nio.DataSink(), name='ds_pdf')
ds_pdf.inputs.base_directory = os.path.join(ds_dir, 'pdfs')
ds_pdf.inputs.parameterization = False
# get atlas data
templates_atlases = {'GM_mask_MNI_2mm': 'SPM_GM/SPM_GM_mask_2mm.nii.gz',
'GM_mask_MNI_3mm': 'SPM_GM/SPM_GM_mask_3mm.nii.gz',
'brain_mask_MNI_3mm': 'cpac_image_resources/MNI_3mm/MNI152_T1_3mm_brain_mask.nii.gz',
'brain_template_MNI_3mm': 'cpac_image_resources/MNI_3mm/MNI152_T1_3mm.nii.gz'
}
selectfiles_anat_templates = Node(nio.SelectFiles(templates_atlases,
base_directory=template_dir),
name="selectfiles_anat_templates")
#####################################
# SET ITERATORS
#####################################
# SUBJECTS ITERATOR
in_data_name_infosource = Node(util.IdentityInterface(fields=['in_data_name']), name='in_data_name_infosource')
in_data_name_infosource.iterables = ('in_data_name', in_data_name_list_unique)
mulitmodal_in_data_name_infosource = Node(util.IdentityInterface(fields=['multimodal_in_data_name']),
name='mulitmodal_in_data_name_infosource')
mulitmodal_in_data_name_infosource.iterables = ('multimodal_in_data_name', in_data_name_list)
subjects_selection_crit_dict = {}
subjects_selection_crit_dict['adult_healthy_F'] = ["df[df.sex == \'F\']", 'df[df.no_axis_1]', 'df[df.age >= 18]']
subjects_selection_crit_dict['adult_F'] = ["df[df.sex == \'F\']", 'df[df.age >= 18]']
subjects_selection_crit_dict['F'] = ["df[df.sex == \'F\']"]
subjects_selection_crit_dict['adult_healthy_M'] = ["df[df.sex == \'M\']", 'df[df.no_axis_1]', 'df[df.age >= 18]']
subjects_selection_crit_dict['adult_M'] = ["df[df.sex == \'M\']", 'df[df.age >= 18]']
subjects_selection_crit_dict['adult'] = ['df[df.age >= 18]']
# subjects_selection_crit_names_list = subjects_selection_crit_dict.keys()
subjects_selection_crit_names_list = ['adult_F']
subject_selection_infosource = Node(util.IdentityInterface(fields=['selection_criterium']),
name='subject_selection_infosource')
subject_selection_infosource.iterables = ('selection_criterium', subjects_selection_crit_names_list)
def out_name_str_fct(selection_criterium, in_data_name):
return selection_criterium + '_' + in_data_name
out_name_str = Node(util.Function(input_names=['selection_criterium', 'in_data_name'],
output_names=['out_name_str'],
function=out_name_str_fct),
name='out_name_str')
wf.connect(in_data_name_infosource, 'in_data_name', out_name_str, 'in_data_name')
wf.connect(subject_selection_infosource, 'selection_criterium', out_name_str, 'selection_criterium')
def get_subjects_info_fct(df_file, subjects_selection_crit_dict, selection_criterium):
import pandas as pd
import os
import numpy as np
df = pd.read_pickle(df_file)
# EXCLUSION HERE:
for eval_str in subjects_selection_crit_dict[selection_criterium]:
df = eval(eval_str)
df_out_file = os.path.join(os.getcwd(), 'df_use.csv')
df.to_csv(df_out_file)
subjects_list = df.leica_id.values
age = df.age.values
age_file = os.path.join(os.getcwd(), 'age.npy')
np.save(age_file, age)
return (age_file, df_out_file, subjects_list)
get_subjects_info = Node(
util.Function(input_names=['df_file', 'subjects_selection_crit_dict', 'selection_criterium'],
output_names=['age_file', 'df_out_file', 'subjects_list'],
function=get_subjects_info_fct),
name='get_subjects_info')
get_subjects_info.inputs.df_file = df_file
get_subjects_info.inputs.subjects_selection_crit_dict = subjects_selection_crit_dict
wf.connect(subject_selection_infosource, 'selection_criterium', get_subjects_info, 'selection_criterium')
wf.connect(get_subjects_info, 'df_out_file', ds, 'test')
wf.connect(get_subjects_info, 'age_file', ds, 'test_age_file')
def create_file_list_fct(subjects_list, in_data_name, data_lookup_dict, brain_mask_path, gm_mask_path):
file_list = []
for s in subjects_list:
file_list.append(data_lookup_dict[in_data_name]['path_str'].format(subject_id=s))
if 'matrix_name' in data_lookup_dict[in_data_name].keys():
matrix_name = data_lookup_dict[in_data_name]['matrix_name']
else:
matrix_name = None
if 'parcellation_path' in data_lookup_dict[in_data_name].keys():
parcellation_path = data_lookup_dict[in_data_name]['parcellation_path']
else:
parcellation_path = None
if 'fwhm' in data_lookup_dict[in_data_name].keys():
fwhm = data_lookup_dict[in_data_name]['fwhm']
else:
fwhm = None
mask_path = brain_mask_path
if 'use_gm_mask' in data_lookup_dict[in_data_name].keys():
if data_lookup_dict[in_data_name]['use_gm_mask']:
mask_path = gm_mask_path
return file_list, matrix_name, parcellation_path, fwhm, mask_path
create_file_list = Node(util.Function(input_names=['subjects_list',
'in_data_name',
'data_lookup_dict',
'brain_mask_path',
'gm_mask_path'],
output_names=['file_list',
'matrix_name',
'parcellation_path',
'fwhm',
'mask_path'],
function=create_file_list_fct),
name='create_file_list')
wf.connect(get_subjects_info, 'subjects_list', create_file_list, 'subjects_list')
wf.connect(in_data_name_infosource, 'in_data_name', create_file_list, 'in_data_name')
create_file_list.inputs.data_lookup_dict = data_lookup_dict
wf.connect(selectfiles_anat_templates, 'brain_mask_MNI_3mm', create_file_list, 'brain_mask_path')
wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', create_file_list, 'gm_mask_path')
aggregate_subjects = Node(util.Function(input_names=['file_list'],
output_names=['merged_file'],
function=aggregate_data),
name='aggregate_subjects')
wf.connect(create_file_list, 'file_list', aggregate_subjects, 'file_list')
vectorized_data = Node(
util.Function(input_names=['in_data_file', 'mask_file', 'matrix_name', 'parcellation_path', 'fwhm'],
output_names=['vectorized_data', 'vectorized_data_file'],
function=vectorize_data),
name='vectorized_data')
wf.connect(aggregate_subjects, 'merged_file', vectorized_data, 'in_data_file')
wf.connect(create_file_list, 'mask_path', vectorized_data, 'mask_file')
wf.connect(create_file_list, 'matrix_name', vectorized_data, 'matrix_name')
wf.connect(create_file_list, 'parcellation_path', vectorized_data, 'parcellation_path')
wf.connect(create_file_list, 'fwhm', vectorized_data, 'fwhm')
def aggregate_multimodal_metrics_fct(multimodal_list, vectorized_data_file, vectorized_data_names,
selection_criterium):
import numpy as np
import os
metrics_index_list = [vectorized_data_names.index(m) for m in multimodal_list]
X = None
for i in metrics_index_list:
X_file = vectorized_data_file[i]
X_single = np.load(X_file)
if X is None:
X = X_single
else:
X = np.hstack((X, X_single))
multimodal_in_name = '_'.join(multimodal_list)
multimodal_out_name = selection_criterium + '_' + multimodal_in_name
X_file = os.path.join(os.getcwd(), multimodal_in_name + '.npy')
np.save(X_file, X)
return multimodal_in_name, multimodal_out_name, X_file
aggregate_multimodal_metrics = JoinNode(util.Function(input_names=['multimodal_list',
'vectorized_data_file',
'vectorized_data_names',
'selection_criterium'],
output_names=['multimodal_in_name', 'multimodal_out_name',
'X_file'],
function=aggregate_multimodal_metrics_fct),
joinfield='vectorized_data_file',
joinsource=in_data_name_infosource,
name='aggregate_multimodal_metrics')
wf.connect(mulitmodal_in_data_name_infosource, 'multimodal_in_data_name', aggregate_multimodal_metrics,
'multimodal_list')
wf.connect(vectorized_data, 'vectorized_data_file', aggregate_multimodal_metrics, 'vectorized_data_file')
wf.connect(subject_selection_infosource, 'selection_criterium', aggregate_multimodal_metrics, 'selection_criterium')
wf.connect(aggregate_multimodal_metrics, 'X_file', ds, 'multimodal_test')
aggregate_multimodal_metrics.inputs.vectorized_data_names = in_data_name_list_unique
def run_prediction(reg_model, X_file, y_file, data_str):
def _pred_real_scatter(y_test, y_test_predicted, title_str, in_data_name):
import os
import pylab as plt
from matplotlib.backends.backend_pdf import PdfPages
plt.scatter(y_test, y_test_predicted)
plt.plot([10, 80], [10, 80], 'k')
plt.xlabel('real')
plt.ylabel('predicted')
ax = plt.gca()
ax.set_aspect('equal')
plt.title(title_str)
plt.tight_layout()
scatter_file = os.path.join(os.getcwd(), 'scatter_' + in_data_name + '.pdf')
pp = PdfPages(scatter_file)
pp.savefig()
pp.close()
return scatter_file
import os, pickle
import numpy as np
from sklearn.svm import SVR
from sklearn.cross_validation import cross_val_score, cross_val_predict
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import Imputer
from sklearn.pipeline import Pipeline
from sklearn.metrics import r2_score, mean_absolute_error
X = np.load(X_file)
y = np.load(y_file)
# fixme pipe
fill_missing = Imputer()
# X = fill_missing.fit_transform(X)
# fixme? pipe
# remove low variance features
var_thr = VarianceThreshold()
# X = var_thr.fit_transform(X)
# fixme pipe normalize
normalize = MinMaxScaler()
# X = normalize.fit_transform(X)
regression_model = reg_model # SVR(kernel='linear')
pipe = Pipeline([
('fill_missing', fill_missing),
('var_thr', var_thr),
('normalize', normalize),
('regression_model', regression_model),
])
# cv_scores = cross_val_score(pipe, X, y, cv=5, n_jobs=5,
# scoring='mean_absolute_error')
# cv_scores_r2 = cross_val_score(pipe, X, y, cv=5, n_jobs=5,
# scoring='r2')
y_predicted = cross_val_predict(pipe, X, y, cv=5, n_jobs=5)
cv_scores = mean_absolute_error(y, y_predicted)
cv_scores_r2 = r2_score(y, y_predicted)
title_str = '{}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}'.format(
data_str,
cv_scores.mean(), cv_scores.std(), cv_scores.min(), cv_scores.max(),
cv_scores_r2.mean(), cv_scores_r2.std(), cv_scores_r2.min(), cv_scores_r2.max())
scatter_file = _pred_real_scatter(y, y_predicted, title_str, data_str)
out_file = os.path.join(os.getcwd(), 'cv_scores.pkl')
with open(out_file, 'w') as f:
pickle.dump(cv_scores, f)
y_pred_file = os.path.join(os.getcwd(), 'y_pred.npy')
np.save(y_pred_file, y_predicted)
return out_file, regression_model, scatter_file, y_pred_file
#
from sklearn.svm import SVR
from sklearn.linear_model import (LassoCV, ElasticNetCV)
prediction = Node(util.Function(input_names=['reg_model', 'X_file', 'y_file', 'data_str'],
output_names=['out_file', 'regression_model', 'scatter_file', 'y_pred_file'],
function=run_prediction),
name='prediction')
# prediction.inputs.reg_model = SVR(kernel='linear')
# wf.connect(aggregate_multimodal_metrics, 'X_file', prediction, 'X_file')
# # wf.connect(vectorized_data, 'vectorized_data_file', prediction, 'X_file')
# wf.connect(get_subjects_info, 'age_file', prediction, 'y_file')
# wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction, 'data_str')
# wf.connect(prediction, 'out_file', ds, 'test_naiv_cv_score')
# wf.connect(prediction, 'scatter_file', ds_pdf, 'test_naive_scatter_svr')
# wf.connect(prediction, 'y_pred_file', ds_pdf, 'test_naive_predicted_svr')
#
# prediction_lasso = prediction.clone('prediction_lasso')
# prediction_lasso.inputs.reg_model = LassoCV(n_jobs=5)
# wf.connect(aggregate_multimodal_metrics, 'X_file', prediction_lasso, 'X_file')
# wf.connect(get_subjects_info, 'age_file', prediction_lasso, 'y_file')
# wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction_lasso, 'data_str')
# wf.connect(prediction_lasso, 'out_file', ds, 'test_cv_score_lasso')
# wf.connect(prediction_lasso, 'scatter_file', ds_pdf, 'test_naive_scatter_lasso')
#
def run_prediction_split(reg_model, X_file, y_file, data_str):
def _pred_real_scatter(y_test, y_test_predicted, title_str, in_data_name):
import os
import pylab as plt
from matplotlib.backends.backend_pdf import PdfPages
plt.scatter(y_test, y_test_predicted)
plt.plot([10, 80], [10, 80], 'k')
plt.xlabel('real')
plt.ylabel('predicted')
ax = plt.gca()
ax.set_aspect('equal')
plt.title(title_str)
plt.tight_layout()
scatter_file = os.path.join(os.getcwd(), 'scatter_' + in_data_name + '.pdf')
pp = PdfPages(scatter_file)
pp.savefig()
pp.close()
return scatter_file
import os, pickle
import numpy as np
from sklearn.svm import SVR
from sklearn.cross_validation import cross_val_score, cross_val_predict
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import Imputer
from sklearn.pipeline import Pipeline
from sklearn.metrics import r2_score, mean_absolute_error
from sklearn.cross_validation import train_test_split
X = np.load(X_file)
y = np.load(y_file)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
fill_missing = Imputer()
var_thr = VarianceThreshold()
normalize = MinMaxScaler()
regression_model = reg_model # SVR(kernel='linear')
pipe = Pipeline([
('fill_missing', fill_missing),
('var_thr', var_thr),
('normalize', normalize),
('regression_model', regression_model),
])
pipe.fit(X_train, y_train)
fitted_enet_model = pipe.steps[3][1]
results_str = 'n_iter: %s. l1_r: %s. alpha: %s' % (
fitted_enet_model.n_iter_, fitted_enet_model.l1_ratio_, fitted_enet_model.alpha_)
y_predicted = pipe.predict(X_test)
cv_scores = mean_absolute_error(y_test, y_predicted)
cv_scores_r2 = r2_score(y_test, y_predicted)
title_str = '{}\n{}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}'.format(
data_str,
results_str,
cv_scores.mean(), cv_scores.std(), cv_scores.min(), cv_scores.max(),
cv_scores_r2.mean(), cv_scores_r2.std(), cv_scores_r2.min(), cv_scores_r2.max())
scatter_file = _pred_real_scatter(y_test, y_predicted, title_str, data_str)
out_file = os.path.join(os.getcwd(), 'cv_scores.pkl')
with open(out_file, 'w') as f:
pickle.dump(cv_scores, f)
model_out_file = os.path.join(os.getcwd(), 'trained_model.pkl')
with open(model_out_file, 'w') as f:
pickle.dump(pipe, f)
y_pred_file = os.path.join(os.getcwd(), 'y_pred.npy')
np.save(y_pred_file, y_predicted)
return out_file, regression_model, scatter_file, y_pred_file, model_out_file, fitted_enet_model
prediction_enet = Node(util.Function(input_names=['reg_model', 'X_file', 'y_file', 'data_str'],
output_names=['out_file', 'regression_model', 'scatter_file',
'y_pred_file', 'model_out_file', 'fitted_enet_model'],
function=run_prediction_split),
name='prediction_enet')
# fixme n_jobs
prediction_enet.inputs.reg_model = ElasticNetCV(cv=5, n_jobs=10, selection='random', max_iter=2000,
l1_ratio=[0, .01, .05, .1, .5, .7, .9, .95, .99, 1])
wf.connect(aggregate_multimodal_metrics, 'X_file', prediction_enet, 'X_file')
wf.connect(get_subjects_info, 'age_file', prediction_enet, 'y_file')
wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction_enet, 'data_str')
wf.connect(prediction_enet, 'out_file', ds, 'test_split_cv_score_enet')
wf.connect(prediction_enet, 'model_out_file', ds, 'test_split_trained_model_enet')
wf.connect(prediction_enet, 'scatter_file', ds_pdf, 'test_split_scatter_enet')
wf.connect(prediction_enet, 'y_pred_file', ds_pdf, 'test_split_predicted_enet')
# prediction_enet = prediction.clone('prediction_enet')
# prediction_enet.inputs.reg_model = ElasticNetCV(cv=5, n_jobs=7, selection='random', max_iter=2000,
# l1_ratio=[.1, .5, .7, .9, .95, .99, 1])
# wf.connect(aggregate_multimodal_metrics, 'X_file', prediction_enet, 'X_file')
# wf.connect(get_subjects_info, 'age_file', prediction_enet, 'y_file')
# wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction_enet, 'data_str')
# wf.connect(prediction_enet, 'out_file', ds, 'test_cv_score_enet')
# wf.connect(prediction_enet, 'scatter_file', ds_pdf, 'test_naive_scatter_enet')
# wf.connect(prediction_enet, 'y_pred_file', ds_pdf, 'test_naive_predicted_enet')
# # # # # # # # # # # # # #
# GS
# # # # # # # # # # # # # #
def run_prediction_gs(X_file, y_file, data_str):
# fixme copied function
def _pred_real_scatter(y_test, y_test_predicted, title_str, in_data_name):
import os
import pylab as plt
from matplotlib.backends.backend_pdf import PdfPages
plt.scatter(y_test, y_test_predicted)
plt.plot([10, 80], [10, 80], 'k')
plt.xlabel('real')
plt.ylabel('predicted')
ax = plt.gca()
ax.set_aspect('equal')
plt.title(title_str)
plt.tight_layout()
scatter_file = os.path.join(os.getcwd(), 'scatter_' + in_data_name + '.pdf')
pp = PdfPages(scatter_file)
pp.savefig()
pp.close()
return scatter_file
import os, pickle
import numpy as np
from sklearn.svm import SVR
from sklearn.cross_validation import cross_val_score, cross_val_predict
from sklearn.feature_selection import VarianceThreshold
from sklearn.grid_search import GridSearchCV
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import Imputer
X = np.load(X_file)
y = np.load(y_file)
# fixme pipe
imp = Imputer()
X = imp.fit_transform(X)
# fixme? pipe
# remove low variance features
var_thr = VarianceThreshold()
X = var_thr.fit_transform(X)
# fixme pipe normalize
normalize = MinMaxScaler()
X = normalize.fit_transform(X)
# fixme?
# remove low variance features
var_thr = VarianceThreshold()
X = var_thr.fit_transform(X)
regression_model = SVR(kernel='linear')
# GRID SEARCH
params = {
'C': [.0001, .001, .01, .1, 1, 10, 20, 30],
'epsilon': [.0001, .001, .01, .1, 1],
}
gs = GridSearchCV(regression_model, params, cv=5, scoring='mean_absolute_error', n_jobs=10)
gs.fit(X, y)
sorted_grid_score = sorted(gs.grid_scores_, key=lambda x: x.mean_validation_score, reverse=True)
score_str = [str(n) + ': ' + str(g) for n, g in enumerate(sorted_grid_score)]
gs_file = os.path.join(os.getcwd(), 'gs_' + data_str + '.txt')
with open(gs_file, 'w') as f:
f.write('\n'.join(score_str))
# GS WITH NESTED CV
gs_cv = GridSearchCV(regression_model, params, cv=5, scoring='mean_absolute_error', n_jobs=5)
cv_scores = cross_val_score(regression_model, X, y, cv=5, scoring='mean_absolute_error')
cv_scores_r2 = cross_val_score(regression_model, X, y, cv=5, scoring='r2')
cv_prediction = cross_val_predict(regression_model, X, y, cv=5)
title_str = '{}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}'.format(
data_str,
cv_scores.mean(), cv_scores.std(), cv_scores.min(), cv_scores.max(),
cv_scores_r2.mean(), cv_scores_r2.std(), cv_scores_r2.min(), cv_scores_r2.max())
cv_scatter_file = _pred_real_scatter(y, cv_prediction, title_str, data_str + '_cv_pred')
## # # # # # # # #
# BEST PARAMETERS
best_params = sorted_grid_score[0].parameters
regression_model = SVR(kernel='linear', **best_params)
y_predicted = cross_val_predict(regression_model, X, y, cv=5, n_jobs=5)
best_params_scatter_file = _pred_real_scatter(y, y_predicted, data_str, data_str)
return regression_model, gs_file, best_params_scatter_file, cv_scatter_file
gs = Node(util.Function(input_names=['X_file', 'y_file', 'data_str'],
output_names=['regression_model', 'gs_file', 'best_params_scatter_file', 'cv_scatter_file'],
function=run_prediction_gs),
name='gs')
# wf.connect(aggregate_multimodal_metrics, 'X_file', gs, 'X_file')
# wf.connect(get_subjects_info, 'age_file', gs, 'y_file')
# wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', gs, 'data_str')
# wf.connect(gs, 'gs_file', ds_pdf, 'test_gs')
# wf.connect(gs, 'best_params_scatter_file', ds_pdf, 'test_gs_best_params_scatters')
# wf.connect(gs, 'cv_scatter_file', ds_pdf, 'test_gs_cv_scatters')
# # # # # # # # # # # # # #
# PIPELINE
# # # # # # # # # # # # # #
def run_prediction_gs_split(X_file, y_file, data_str):
# fixme copied function
def _pred_real_scatter(y_test, y_test_predicted, title_str, in_data_name):
import os
import pylab as plt
from matplotlib.backends.backend_pdf import PdfPages
plt.scatter(y_test, y_test_predicted)
plt.plot([10, 80], [10, 80], 'k')
plt.xlabel('real')
plt.ylabel('predicted')
ax = plt.gca()
ax.set_aspect('equal')
plt.title(title_str)
plt.tight_layout()
scatter_file = os.path.join(os.getcwd(), 'scatter_' + in_data_name + '.pdf')
pp = PdfPages(scatter_file)
pp.savefig()
pp.close()
return scatter_file
import os, pickle
import numpy as np
from sklearn.svm import SVR
from sklearn.cross_validation import cross_val_score, cross_val_predict, train_test_split
from sklearn.grid_search import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.preprocessing import Imputer
from sklearn.feature_selection import SelectPercentile, f_regression
from sklearn.metrics import mean_absolute_error, r2_score
from sklearn.svm import LinearSVR
from sklearn.decomposition import PCA
X = np.load(X_file)
y = np.load(y_file)
# fixme add squared values to X
# X = np.hstack([X, np.square(X)])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
# remove low variance features
fill_missing = Imputer()
var_thr = VarianceThreshold()
normalize = StandardScaler() # MinMaxScaler()
selection = SelectPercentile(f_regression)
# regression_model = LinearSVR() #SVR(kernel='linear')
from sklearn.svm import NuSVR
regression_model = LinearSVR() # NuSVR(kernel='linear') #SVR(kernel='linear')
pipe = Pipeline([
('fill_missing', fill_missing),
('var_thr', var_thr),
('normalize', normalize),
('selection', selection),
('regression_model', regression_model),
])
# GRID SEARCH
params = {
'selection__percentile': [70, 75, 80, 85, 90, 95, 99, 100],
'regression_model__C': [1, 98, 106, 110, 130, 150, 170, 200, 14450],
# [.0001, .001, .01, .1, 1, 10, 20, 30],
'regression_model__epsilon': [0, .005, .01, .05, .1, 1, 5, 10],
'regression_model__loss': ['epsilon_insensitive', 'squared_epsilon_insensitive'],
# 'regression_model__nu': [0.01, .25, .5, .75, .8, .85, .9, .95, .99],
}
# fixme njobs
gs = GridSearchCV(pipe, params, cv=5, scoring='mean_absolute_error', n_jobs=30)
gs.fit(X_train, y_train)
best_estimator = gs.best_estimator_
grid_scores = gs.grid_scores_
gs_file = os.path.join(os.getcwd(), 'gs_' + data_str + '.pkl')
with open(gs_file, 'w') as f:
pickle.dump(grid_scores, f)
sorted_grid_score = sorted(gs.grid_scores_, key=lambda x: x.mean_validation_score, reverse=True)
score_str = [str(n) + ': ' + str(g) for n, g in enumerate(sorted_grid_score)]
gs_text_file = os.path.join(os.getcwd(), 'gs_txt_' + data_str + '.txt')
with open(gs_text_file, 'w') as f:
f.write('\n'.join(score_str))
# fitted_model = gs.steps[-1][1]
model_out_file = os.path.join(os.getcwd(), 'trained_model.pkl')
with open(model_out_file, 'w') as f:
pickle.dump(gs, f)
y_predicted = gs.predict(X_test)
cv_scores = mean_absolute_error(y_test, y_predicted)
cv_scores_r2 = r2_score(y_test, y_predicted)
title_str = '{}\n mae: {:.3f}\n r2: {:.3f}'.format(
data_str,
cv_scores,
cv_scores_r2)
scatter_file = _pred_real_scatter(y_test, y_predicted, title_str, data_str)
return model_out_file, scatter_file, gs_text_file, gs_file, best_estimator
prediction_gs_split = Node(util.Function(input_names=['X_file', 'y_file', 'data_str'],
output_names=['model_out_file', 'scatter_file', 'gs_text_file', 'gs_file',
'best_estimator'],
function=run_prediction_gs_split),
name='prediction_gs_split')
wf.connect(aggregate_multimodal_metrics, 'X_file', prediction_gs_split, 'X_file')
wf.connect(get_subjects_info, 'age_file', prediction_gs_split, 'y_file')
wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction_gs_split, 'data_str')
wf.connect(prediction_gs_split, 'scatter_file', ds_pdf, 'test_gs_split_scatter')
wf.connect(prediction_gs_split, 'model_out_file', ds, 'test_gs_split_trained_model')
wf.connect(prediction_gs_split, 'gs_text_file', ds_pdf, 'test_gs_split_gs_params')
wf.connect(prediction_gs_split, 'gs_file', ds_pdf, 'grid_scores')
# RANDOM CV
def run_prediction_random_gs_split(X_file, y_file, data_str):
# fixme copied function
def _pred_real_scatter(y_test, y_test_predicted, title_str, in_data_name):
import os
import pylab as plt
from matplotlib.backends.backend_pdf import PdfPages
plt.scatter(y_test, y_test_predicted)
plt.plot([10, 80], [10, 80], 'k')
plt.xlabel('real')
plt.ylabel('predicted')
ax = plt.gca()
ax.set_aspect('equal')
plt.title(title_str)
plt.tight_layout()
scatter_file = os.path.join(os.getcwd(), 'scatter_' + in_data_name + '.pdf')
pp = PdfPages(scatter_file)
pp.savefig()
pp.close()
return scatter_file
import os, pickle
import numpy as np
from sklearn.svm import SVR
from sklearn.cross_validation import cross_val_score, cross_val_predict, train_test_split
from sklearn.grid_search import RandomizedSearchCV
from sklearn.pipeline import Pipeline
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.preprocessing import Imputer
from sklearn.feature_selection import SelectPercentile, f_regression
from sklearn.metrics import mean_absolute_error, r2_score
from sklearn.svm import LinearSVR
from sklearn.decomposition import PCA
from scipy.stats import randint as sp_randint
from scipy.stats import expon
X = np.load(X_file)
y = np.load(y_file)
# fixme add squared values to X
# X = np.hstack([X, np.square(X)])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
# remove low variance features
fill_missing = Imputer()
var_thr = VarianceThreshold()
normalize = StandardScaler() # MinMaxScaler()
selection = SelectPercentile(f_regression)
# regression_model = LinearSVR() #SVR(kernel='linear')
from sklearn.svm import NuSVR
regression_model = LinearSVR() # NuSVR(kernel='linear') #SVR(kernel='linear')
pipe = Pipeline([
('fill_missing', fill_missing),
('var_thr', var_thr),
('normalize', normalize),
('selection', selection),
('regression_model', regression_model),
])
param_dist = {
'selection__percentile': sp_randint(10, 100),
'regression_model__C': expon(scale=100), # sp_randint(.001, 14450),
'regression_model__epsilon': sp_randint(0, 100),
'regression_model__loss': ['epsilon_insensitive', 'squared_epsilon_insensitive'],
}
# fixme njobs
n_iter_search = 400
gs = RandomizedSearchCV(pipe, param_distributions=param_dist, cv=5, scoring='mean_absolute_error', n_jobs=15, n_iter=n_iter_search)
gs.fit(X_train, y_train)
best_estimator = gs.best_estimator_
grid_scores = gs.grid_scores_
gs_file = os.path.join(os.getcwd(), 'gs_' + data_str + '.pkl')
with open(gs_file, 'w') as f:
pickle.dump(grid_scores, f)
sorted_grid_score = sorted(gs.grid_scores_, key=lambda x: x.mean_validation_score, reverse=True)
score_str = [str(n) + ': ' + str(g) for n, g in enumerate(sorted_grid_score)]
gs_text_file = os.path.join(os.getcwd(), 'gs_txt_' + data_str + '.txt')
with open(gs_text_file, 'w') as f:
f.write('\n'.join(score_str))
# fitted_model = gs.steps[-1][1]
# fixme pickle crashes
model_out_file = ''
# model_out_file = os.path.join(os.getcwd(), 'trained_model.pkl')
# with open(model_out_file, 'w') as f:
# pickle.dump(gs, f)
y_predicted = gs.predict(X_test)
cv_scores = mean_absolute_error(y_test, y_predicted)
cv_scores_r2 = r2_score(y_test, y_predicted)
title_str = '{}\n mae: {:.3f}\n r2: {:.3f}'.format(
data_str,
cv_scores,
cv_scores_r2)
scatter_file = _pred_real_scatter(y_test, y_predicted, title_str, data_str)
return model_out_file, scatter_file, gs_text_file, gs_file, best_estimator
prediction_random_gs_split = Node(util.Function(input_names=['X_file', 'y_file', 'data_str'],
output_names=['model_out_file', 'scatter_file', 'gs_text_file', 'gs_file',
'best_estimator'],
function=run_prediction_random_gs_split),
name='prediction_random_gs_split')
wf.connect(aggregate_multimodal_metrics, 'X_file', prediction_random_gs_split, 'X_file')
wf.connect(get_subjects_info, 'age_file', prediction_random_gs_split, 'y_file')
wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction_random_gs_split, 'data_str')
wf.connect(prediction_random_gs_split, 'scatter_file', ds_pdf, 'test_random_gs_split_scatter')
#wf.connect(prediction_random_gs_split, 'model_out_file', ds, 'test_random_gs_split_trained_model')
wf.connect(prediction_random_gs_split, 'gs_text_file', ds_pdf, 'test_random_gs_split_gs_params')
wf.connect(prediction_random_gs_split, 'gs_file', ds_pdf, 'random_grid_scores')
from learning_curve import learning_curve_fct
learning_curve_svr = Node(util.Function(input_names=['X_file', 'y_file', 'out_name', 'best_estimator'],
output_names=['curve_file'],
function=learning_curve_fct),
name='learning_curve_svr')
wf.connect(aggregate_multimodal_metrics, 'X_file', learning_curve_svr, 'X_file')
wf.connect(get_subjects_info, 'age_file', learning_curve_svr, 'y_file')
wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', learning_curve_svr, 'out_name')
wf.connect(prediction_gs_split, 'best_estimator', learning_curve_svr, 'best_estimator')
# wf.connect(prediction_gs_split, 'model_out_file', learning_curve_svr, 'fitted_model_file')
wf.connect(learning_curve_svr, 'curve_file', ds_pdf, 'learning_curve_svr')
def _create_best_enet(fitted_enet):
from sklearn.linear_model import ElasticNet
best_estimator = ElasticNet(selection='random', max_iter=2000,
alpha=fitted_enet.alpha_,
l1_ratio=fitted_enet.l1_ratio_)
return best_estimator
best_estimator_enet = Node(util.Function(input_names=['fitted_enet'],
output_names=['best_estimator'], function=_create_best_enet),
name='best_estimator_enet')
wf.connect(prediction_enet, 'fitted_enet_model', best_estimator_enet, 'fitted_enet')
learning_curve_enet = Node(util.Function(input_names=['X_file', 'y_file', 'out_name', 'best_estimator'],
output_names=['curve_file'],
function=learning_curve_fct),
name='learning_curve_enet')
wf.connect(aggregate_multimodal_metrics, 'X_file', learning_curve_enet, 'X_file')
wf.connect(get_subjects_info, 'age_file', learning_curve_enet, 'y_file')
wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', learning_curve_enet, 'out_name')
wf.connect(best_estimator_enet, 'best_estimator', learning_curve_enet, 'best_estimator')
wf.connect(learning_curve_enet, 'curve_file', ds_pdf, 'learning_curve_enet')
def plot_grid_scores_fct(gs_file, data_str):
import pylab as plt
import pickle, os
import pandas as pd
import numpy as np
with open(gs_file) as fi:
gs = pickle.load(fi)
# build data frame
names_params = gs[0].parameters.keys()
c_names = names_params + ['m', 'sd']
df = pd.DataFrame([], columns=c_names)
for g in gs:
data_params = [g.parameters[n] for n in names_params]
data_m = [np.abs(g.mean_validation_score), g.cv_validation_scores.std()]
data = data_params + data_m
df.loc[len(df)] = data
# plot
plt.figure(figsize=(5, 4 * len(names_params)))
for i, p in enumerate(names_params):
plt.subplot(len(names_params), 1, i + 1)
x = df[p].values
y = df['m'].values
if type(x[0]) is str:
rep_dict = {}
for i, o in enumerate(np.unique(x)):
rep_dict[o] = i
x = np.array([rep_dict[o] for o in x])
x_ticks = rep_dict.keys()
x_vals = map(rep_dict.get, x_ticks)
plt.xticks(x_vals, x_ticks, rotation='vertical')
u_x = x_vals
u_y = [np.mean(df['m'][df[p] == v]) for v in rep_dict.keys()]
else:
u_x = np.unique(x)
u_y = [np.mean(df['m'][df[p] == v]) for v in u_x]
plt.scatter(x, y, color='black')
plt.plot(u_x, u_y)
plt.scatter(u_x, u_y, marker='+')
plt.title(p)
plt.tight_layout()
fig_file = os.path.join(os.getcwd(), 'gs_plot_' + data_str + '.pdf')
plt.savefig(fig_file)
return fig_file
plot_grid_scores = Node(util.Function(input_names=['gs_file', 'data_str'],
output_names=['fig_file'],
function=plot_grid_scores_fct),
name='plot_grid_scores')
wf.connect(prediction_gs_split, 'gs_file', plot_grid_scores, 'gs_file')
wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', plot_grid_scores, 'data_str')
wf.connect(plot_grid_scores, 'fig_file', ds_pdf, 'gs_grid_score_plots')
plot_random_grid_scores = Node(util.Function(input_names=['gs_file', 'data_str'],
output_names=['fig_file'],
function=plot_grid_scores_fct),
name='plot_random_grid_scores')
wf.connect(prediction_random_gs_split, 'gs_file', plot_random_grid_scores, 'gs_file')
wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', plot_random_grid_scores, 'data_str')
wf.connect(plot_random_grid_scores, 'fig_file', ds_pdf, 'gs_random_grid_score_plots')
def plot_validation_curve_fct(X_file, y_file, best_estimator, data_str):
import pylab as plt
import pickle, os
import pandas as pd
import numpy as np
from sklearn.learning_curve import validation_curve
from matplotlib.backends.backend_pdf import PdfPages
X = np.load(X_file)
y = np.load(y_file)
params = {
'selection__percentile': [70, 75, 80, 85, 90, 95, 99, 100],
'regression_model__C': [1, 98, 106, 110, 130, 150, 170, 200, 1000, 10000, 14450],
# [.0001, .001, .01, .1, 1, 10, 20, 30],
'regression_model__epsilon': [0, .005, .01, .05, .1, 1, 5, 10],
'regression_model__loss': ['epsilon_insensitive', 'squared_epsilon_insensitive'],
# 'regression_model__nu': [0.01, .25, .5, .75, .8, .85, .9, .95, .99],
}
fig_file = os.path.abspath('validation_curve_' + data_str + '.pdf')
fig_file_log = os.path.abspath('validation_curve_log' + data_str + '.pdf')
with PdfPages(fig_file, 'w') as pdf:
with PdfPages(fig_file_log, 'w') as pdf_log:
for p in sorted(params.keys()):
param_range = params[p]
if type(param_range[0]) is not str:
train_scores, test_scores = validation_curve(best_estimator, X, y, param_name=p,
param_range=param_range,
cv=5, n_jobs=5)
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)
# LINEAR AXIS
plt.title('Validation Curve')
plt.xlabel(p)
plt.ylim(0.0, 1.1)
plt.plot(param_range, train_scores_mean, label='Training score', color='r')
plt.fill_between(param_range, train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std,
alpha=0.2, color='r')
plt.plot(param_range, test_scores_mean, label='Cross-validation score', color='g')
plt.fill_between(param_range, test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std,
alpha=0.2, color='g')
plt.legend(loc='best')
pdf.savefig()
plt.close()
# LOG AXIS
plt.title('Validation Curve')
plt.xlabel(p)
plt.ylim(0.0, 1.1)
plt.semilogx(param_range, train_scores_mean, label='Training score', color='r')
plt.fill_between(param_range, train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std,
alpha=0.2, color='r')
plt.semilogx(param_range, test_scores_mean, label='Cross-validation score', color='g')
plt.fill_between(param_range, test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std,
alpha=0.2, color='g')
plt.legend(loc='best')
pdf_log.savefig()
plt.close()
return fig_file, fig_file_log
plot_validation_curve = Node(util.Function(input_names=['X_file', 'y_file', 'best_estimator', 'data_str'],
output_names=['fig_file', 'fig_file_log'],
function=plot_validation_curve_fct),
name='plot_validation_curve')
wf.connect(aggregate_multimodal_metrics, 'X_file', plot_validation_curve, 'X_file')
wf.connect(get_subjects_info, 'age_file', plot_validation_curve, 'y_file')
wf.connect(prediction_gs_split, 'best_estimator', plot_validation_curve, 'best_estimator')
wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', plot_validation_curve, 'data_str')
wf.connect(plot_validation_curve, 'fig_file', ds_pdf, 'validation_curve.@lin')
wf.connect(plot_validation_curve, 'fig_file_log', ds_pdf, 'validation_curve.@log')
def run_pca_fct(X_file, data_str):
from sklearn.decomposition import PCA
import numpy as np
import pylab as plt
import os
from sklearn.feature_selection import VarianceThreshold
from sklearn.grid_search import GridSearchCV
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import Imputer
X = np.load(X_file)
# fixme pipe
imp = Imputer()
X = imp.fit_transform(X)
# fixme? pipe
# remove low variance features
var_thr = VarianceThreshold()
X = var_thr.fit_transform(X)
# fixme pipe normalize
normalize = MinMaxScaler()
X = normalize.fit_transform(X)
# fixme?
# remove low variance features
var_thr = VarianceThreshold()
X = var_thr.fit_transform(X)
pca = PCA()
p = pca.fit_transform(X)
explained_var = sum(pca.explained_variance_ratio_)
plt.plot(range(1, len(pca.explained_variance_ratio_) + 1), np.cumsum(pca.explained_variance_ratio_))
plt.title('explained var: %s' % explained_var)
plt.xlabel('n_components')
plt.tight_layout()
out_fig = os.path.join(os.getcwd(), 'pca_var_%s.pdf' % data_str)
plt.savefig(out_fig)
return out_fig
run_pca = Node(util.Function(input_names=['X_file', 'data_str'],
output_names=['out_fig'],
function=run_pca_fct),
name='run_pca')
wf.connect(aggregate_multimodal_metrics, 'X_file', run_pca, 'X_file')
wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', run_pca, 'data_str')
wf.connect(run_pca, 'out_fig', ds_pdf, 'pca')
#####################################
# RUN WF
#####################################
wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical')
wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf')
wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf')
if plugin_name == 'CondorDAGMan':
wf.run(plugin=plugin_name)
if plugin_name == 'MultiProc':
wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def _run_workflow(args):
# build pipeline for each subject, individually
# ~ 5 min 20 sec per subject
# (roughly 320 seconds)
import os
import os.path as op
import sys
import nipype.interfaces.io as nio
import nipype.pipeline.engine as pe
import nipype.interfaces.utility as util
import nipype.interfaces.fsl.maths as fsl
import glob
import time
from time import strftime
from nipype import config as nyconfig
resource_pool, config, subject_info, run_name, site_name = args
sub_id = str(subject_info[0])
qap_type = config['qap_type']
if subject_info[1]:
session_id = subject_info[1]
else:
session_id = "session_0"
if subject_info[2]:
scan_id = subject_info[2]
else:
scan_id = "scan_0"
# Read and apply general settings in config
keep_outputs = config.get('write_all_outputs', False)
output_dir = op.join(config["output_directory"], run_name,
sub_id, session_id, scan_id)
try:
os.makedirs(output_dir)
except:
if not op.isdir(output_dir):
err = "[!] Output directory unable to be created.\n" \
"Path: %s\n\n" % output_dir
raise Exception(err)
else:
pass
log_dir = output_dir
# set up logging
nyconfig.update_config(
{'logging': {'log_directory': log_dir, 'log_to_file': True}})
logging.update_logging(nyconfig)
# take date+time stamp for run identification purposes
unique_pipeline_id = strftime("%Y%m%d%H%M%S")
pipeline_start_stamp = strftime("%Y-%m-%d_%H:%M:%S")
pipeline_start_time = time.time()
logger.info("Pipeline start time: %s" % pipeline_start_stamp)
logger.info("Contents of resource pool:\n" + str(resource_pool))
logger.info("Configuration settings:\n" + str(config))
# for QAP spreadsheet generation only
config.update({"subject_id": sub_id, "session_id": session_id,
"scan_id": scan_id, "run_name": run_name})
if site_name:
config["site_name"] = site_name
workflow = pe.Workflow(name=scan_id)
workflow.base_dir = op.join(config["working_directory"], sub_id,
session_id)
# set up crash directory
workflow.config['execution'] = \
{'crashdump_dir': config["output_directory"]}
# update that resource pool with what's already in the output directory
for resource in os.listdir(output_dir):
if (op.isdir(op.join(output_dir, resource)) and
resource not in resource_pool.keys()):
resource_pool[resource] = glob.glob(op.join(output_dir,
resource, "*"))[0]
# resource pool check
invalid_paths = []
for resource in resource_pool.keys():
if not op.isfile(resource_pool[resource]):
invalid_paths.append((resource, resource_pool[resource]))
if len(invalid_paths) > 0:
err = "\n\n[!] The paths provided in the subject list to the " \
"following resources are not valid:\n"
for path_tuple in invalid_paths:
err = err + path_tuple[0] + ": " + path_tuple[1] + "\n"
err = err + "\n\n"
raise Exception(err)
# start connecting the pipeline
if 'qap_' + qap_type not in resource_pool.keys():
from qap import qap_workflows as qw
wf_builder = getattr(qw, 'qap_' + qap_type + '_workflow')
workflow, resource_pool = wf_builder(workflow, resource_pool, config)
# set up the datasinks
new_outputs = 0
out_list = ['qap_' + qap_type]
if keep_outputs:
out_list = resource_pool.keys()
# Save reports to out_dir if necessary
if config.get('write_report', False):
out_list += ['qap_mosaic']
# The functional temporal also has an FD plot
if 'functional_temporal' in qap_type:
out_list += ['qap_fd']
for output in out_list:
# we use a check for len()==2 here to select those items in the
# resource pool which are tuples of (node, node_output), instead
# of the items which are straight paths to files
# resource pool items which are in the tuple format are the
# outputs that have been created in this workflow because they
# were not present in the subject list YML (the starting resource
# pool) and had to be generated
if len(resource_pool[output]) == 2:
ds = pe.Node(nio.DataSink(), name='datasink_%s' % output)
ds.inputs.base_directory = output_dir
node, out_file = resource_pool[output]
workflow.connect(node, out_file, ds, output)
new_outputs += 1
rt = {'id': sub_id, 'session': session_id, 'scan': scan_id,
'status': 'started'}
# run the pipeline (if there is anything to do)
if new_outputs > 0:
workflow.write_graph(
dotfilename=op.join(output_dir, run_name + ".dot"),
simple_form=False)
nc_per_subject = config.get('num_cores_per_subject', 1)
runargs = {'plugin': 'Linear', 'plugin_args': {}}
if nc_per_subject > 1:
runargs['plugin'] = 'MultiProc',
runargs['plugin_args'] = {'n_procs': nc_per_subject}
try:
workflow.run(**runargs)
rt['status'] = 'finished'
except Exception as e: # TODO We should be more specific here ...
rt.update({'status': 'failed', 'msg': e.msg})
# ... however this is run inside a pool.map: do not raise Execption
else:
rt['status'] = 'cached'
logger.info("\nEverything is already done for subject %s." % sub_id)
# Remove working directory when done
if not keep_outputs:
try:
work_dir = op.join(workflow.base_dir, scan_id)
if op.exists(work_dir):
import shutil
shutil.rmtree(work_dir)
except:
logger.warn("Couldn\'t remove the working directory!")
pass
pipeline_end_stamp = strftime("%Y-%m-%d_%H:%M:%S")
pipeline_end_time = time.time()
logger.info("Elapsed time (minutes) since last start: %s"
% ((pipeline_end_time - pipeline_start_time) / 60))
logger.info("Pipeline end time: %s" % pipeline_end_stamp)
return rt
def builder(subject_id,
subId,
project_dir,
data_dir,
output_dir,
output_final_dir,
output_interm_dir,
layout,
anat=None,
funcs=None,
fmaps=None,
task_name='',
session=None,
apply_trim=False,
apply_dist_corr=False,
apply_smooth=False,
apply_filter=False,
mni_template='2mm',
apply_n4=True,
ants_threads=8,
readable_crash_files=False,
write_logs=True):
"""
Core function that returns a workflow. See wfmaker for more details.
Args:
subject_id: name of subject folder for final outputted sub-folder name
subId: abbreviate name of subject for intermediate outputted sub-folder name
project_dir: full path to root of project
data_dir: full path to raw data files
output_dir: upper level output dir (others will be nested within this)
output_final_dir: final preprocessed sub-dir name
output_interm_dir: intermediate preprcess sub-dir name
layout: BIDS layout instance
"""
##################
### PATH SETUP ###
##################
if session is not None:
session = int(session)
if session < 10:
session = '0' + str(session)
else:
session = str(session)
# Set MNI template
MNItemplate = os.path.join(get_resource_path(),
'MNI152_T1_' + mni_template + '_brain.nii.gz')
MNImask = os.path.join(get_resource_path(),
'MNI152_T1_' + mni_template + '_brain_mask.nii.gz')
MNItemplatehasskull = os.path.join(get_resource_path(),
'MNI152_T1_' + mni_template + '.nii.gz')
# Set ANTs files
bet_ants_template = os.path.join(get_resource_path(),
'OASIS_template.nii.gz')
bet_ants_prob_mask = os.path.join(
get_resource_path(), 'OASIS_BrainCerebellumProbabilityMask.nii.gz')
bet_ants_registration_mask = os.path.join(
get_resource_path(), 'OASIS_BrainCerebellumRegistrationMask.nii.gz')
#################################
### NIPYPE IMPORTS AND CONFIG ###
#################################
# Update nipype global config because workflow.config[] = ..., doesn't seem to work
# Can't store nipype config/rc file in container anyway so set them globaly before importing and setting up workflow as suggested here: http://nipype.readthedocs.io/en/latest/users/config_file.html#config-file
# Create subject's intermediate directory before configuring nipype and the workflow because that's where we'll save log files in addition to intermediate files
if not os.path.exists(os.path.join(output_interm_dir, subId, 'logs')):
os.makedirs(os.path.join(output_interm_dir, subId, 'logs'))
log_dir = os.path.join(output_interm_dir, subId, 'logs')
from nipype import config
if readable_crash_files:
cfg = dict(execution={'crashfile_format': 'txt'})
config.update_config(cfg)
config.update_config({
'logging': {
'log_directory': log_dir,
'log_to_file': write_logs
},
'execution': {
'crashdump_dir': log_dir
}
})
from nipype import logging
logging.update_logging(config)
# Now import everything else
from nipype.interfaces.io import DataSink
from nipype.interfaces.utility import Merge, IdentityInterface
from nipype.pipeline.engine import Node, Workflow
from nipype.interfaces.nipy.preprocess import ComputeMask
from nipype.algorithms.rapidart import ArtifactDetect
from nipype.interfaces.ants.segmentation import BrainExtraction, N4BiasFieldCorrection
from nipype.interfaces.ants import Registration, ApplyTransforms
from nipype.interfaces.fsl import MCFLIRT, TOPUP, ApplyTOPUP
from nipype.interfaces.fsl.maths import MeanImage
from nipype.interfaces.fsl import Merge as MERGE
from nipype.interfaces.fsl.utils import Smooth
from nipype.interfaces.nipy.preprocess import Trim
from .interfaces import Plot_Coregistration_Montage, Plot_Quality_Control, Plot_Realignment_Parameters, Create_Covariates, Down_Sample_Precision, Create_Encoding_File, Filter_In_Mask
##################
### INPUT NODE ###
##################
# Turn functional file list into interable Node
func_scans = Node(IdentityInterface(fields=['scan']), name='func_scans')
func_scans.iterables = ('scan', funcs)
# Get TR for use in filtering below; we're assuming all BOLD runs have the same TR
tr_length = layout.get_metadata(funcs[0])['RepetitionTime']
#####################################
## TRIM ##
#####################################
if apply_trim:
trim = Node(Trim(), name='trim')
trim.inputs.begin_index = apply_trim
#####################################
## DISTORTION CORRECTION ##
#####################################
if apply_dist_corr:
# Get fmap file locations
fmaps = [
f.filename for f in layout.get(
subject=subId, modality='fmap', extensions='.nii.gz')
]
if not fmaps:
raise IOError(
"Distortion Correction requested but field map scans not found..."
)
# Get fmap metadata
totalReadoutTimes, measurements, fmap_pes = [], [], []
for i, fmap in enumerate(fmaps):
# Grab total readout time for each fmap
totalReadoutTimes.append(
layout.get_metadata(fmap)['TotalReadoutTime'])
# Grab measurements (for some reason pyBIDS doesn't grab dcm_meta... fields from side-car json file and json.load, doesn't either; so instead just read the header using nibabel to determine number of scans)
measurements.append(nib.load(fmap).header['dim'][4])
# Get phase encoding direction
fmap_pe = layout.get_metadata(fmap)["PhaseEncodingDirection"]
fmap_pes.append(fmap_pe)
encoding_file_writer = Node(interface=Create_Encoding_File(),
name='create_encoding')
encoding_file_writer.inputs.totalReadoutTimes = totalReadoutTimes
encoding_file_writer.inputs.fmaps = fmaps
encoding_file_writer.inputs.fmap_pes = fmap_pes
encoding_file_writer.inputs.measurements = measurements
encoding_file_writer.inputs.file_name = 'encoding_file.txt'
merge_to_file_list = Node(interface=Merge(2),
infields=['in1', 'in2'],
name='merge_to_file_list')
merge_to_file_list.inputs.in1 = fmaps[0]
merge_to_file_list.inputs.in1 = fmaps[1]
# Merge AP and PA distortion correction scans
merger = Node(interface=MERGE(dimension='t'), name='merger')
merger.inputs.output_type = 'NIFTI_GZ'
merger.inputs.in_files = fmaps
merger.inputs.merged_file = 'merged_epi.nii.gz'
# Create distortion correction map
topup = Node(interface=TOPUP(), name='topup')
topup.inputs.output_type = 'NIFTI_GZ'
# Apply distortion correction to other scans
apply_topup = Node(interface=ApplyTOPUP(), name='apply_topup')
apply_topup.inputs.output_type = 'NIFTI_GZ'
apply_topup.inputs.method = 'jac'
apply_topup.inputs.interp = 'spline'
###################################
### REALIGN ###
###################################
realign_fsl = Node(MCFLIRT(), name="realign")
realign_fsl.inputs.cost = 'mutualinfo'
realign_fsl.inputs.mean_vol = True
realign_fsl.inputs.output_type = 'NIFTI_GZ'
realign_fsl.inputs.save_mats = True
realign_fsl.inputs.save_rms = True
realign_fsl.inputs.save_plots = True
###################################
### MEAN EPIs ###
###################################
# For coregistration after realignment
mean_epi = Node(MeanImage(), name='mean_epi')
mean_epi.inputs.dimension = 'T'
# For after normalization is done to plot checks
mean_norm_epi = Node(MeanImage(), name='mean_norm_epi')
mean_norm_epi.inputs.dimension = 'T'
###################################
### MASK, ART, COV CREATION ###
###################################
compute_mask = Node(ComputeMask(), name='compute_mask')
compute_mask.inputs.m = .05
art = Node(ArtifactDetect(), name='art')
art.inputs.use_differences = [True, False]
art.inputs.use_norm = True
art.inputs.norm_threshold = 1
art.inputs.zintensity_threshold = 3
art.inputs.mask_type = 'file'
art.inputs.parameter_source = 'FSL'
make_cov = Node(Create_Covariates(), name='make_cov')
################################
### N4 BIAS FIELD CORRECTION ###
################################
if apply_n4:
n4_correction = Node(N4BiasFieldCorrection(), name='n4_correction')
n4_correction.inputs.copy_header = True
n4_correction.inputs.save_bias = False
n4_correction.inputs.num_threads = ants_threads
n4_correction.inputs.input_image = anat
###################################
### BRAIN EXTRACTION ###
###################################
brain_extraction_ants = Node(BrainExtraction(), name='brain_extraction')
brain_extraction_ants.inputs.dimension = 3
brain_extraction_ants.inputs.use_floatingpoint_precision = 1
brain_extraction_ants.inputs.num_threads = ants_threads
brain_extraction_ants.inputs.brain_probability_mask = bet_ants_prob_mask
brain_extraction_ants.inputs.keep_temporary_files = 1
brain_extraction_ants.inputs.brain_template = bet_ants_template
brain_extraction_ants.inputs.extraction_registration_mask = bet_ants_registration_mask
brain_extraction_ants.inputs.out_prefix = 'bet'
###################################
### COREGISTRATION ###
###################################
coregistration = Node(Registration(), name='coregistration')
coregistration.inputs.float = False
coregistration.inputs.output_transform_prefix = "meanEpi2highres"
coregistration.inputs.transforms = ['Rigid']
coregistration.inputs.transform_parameters = [(0.1, ), (0.1, )]
coregistration.inputs.number_of_iterations = [[1000, 500, 250, 100]]
coregistration.inputs.dimension = 3
coregistration.inputs.num_threads = ants_threads
coregistration.inputs.write_composite_transform = True
coregistration.inputs.collapse_output_transforms = True
coregistration.inputs.metric = ['MI']
coregistration.inputs.metric_weight = [1]
coregistration.inputs.radius_or_number_of_bins = [32]
coregistration.inputs.sampling_strategy = ['Regular']
coregistration.inputs.sampling_percentage = [0.25]
coregistration.inputs.convergence_threshold = [1e-08]
coregistration.inputs.convergence_window_size = [10]
coregistration.inputs.smoothing_sigmas = [[3, 2, 1, 0]]
coregistration.inputs.sigma_units = ['mm']
coregistration.inputs.shrink_factors = [[4, 3, 2, 1]]
coregistration.inputs.use_estimate_learning_rate_once = [True]
coregistration.inputs.use_histogram_matching = [False]
coregistration.inputs.initial_moving_transform_com = True
coregistration.inputs.output_warped_image = True
coregistration.inputs.winsorize_lower_quantile = 0.01
coregistration.inputs.winsorize_upper_quantile = 0.99
###################################
### NORMALIZATION ###
###################################
# Settings Explanations
# Only a few key settings are worth adjusting and most others relate to how ANTs optimizer starts or iterates and won't make a ton of difference
# Brian Avants referred to these settings as the last "best tested" when he was aligning fMRI data: https://github.com/ANTsX/ANTsRCore/blob/master/R/antsRegistration.R#L275
# Things that matter the most:
# smoothing_sigmas:
# how much gaussian smoothing to apply when performing registration, probably want the upper limit of this to match the resolution that the data is collected at e.g. 3mm
# Old settings [[3,2,1,0]]*3
# shrink_factors
# The coarseness with which to do registration
# Old settings [[8,4,2,1]] * 3
# >= 8 may result is some problems causing big chunks of cortex with little fine grain spatial structure to be moved to other parts of cortex
# Other settings
# transform_parameters:
# how much regularization to do for fitting that transformation
# for syn this pertains to both the gradient regularization term, and the flow, and elastic terms. Leave the syn settings alone as they seem to be the most well tested across published data sets
# radius_or_number_of_bins
# This is the bin size for MI metrics and 32 is probably adequate for most use cases. Increasing this might increase precision (e.g. to 64) but takes exponentially longer
# use_histogram_matching
# Use image intensity distribution to guide registration
# Leave it on for within modality registration (e.g. T1 -> MNI), but off for between modality registration (e.g. EPI -> T1)
# convergence_threshold
# threshold for optimizer
# convergence_window_size
# how many samples should optimizer average to compute threshold?
# sampling_strategy
# what strategy should ANTs use to initialize the transform. Regular here refers to approximately random sampling around the center of the image mass
normalization = Node(Registration(), name='normalization')
normalization.inputs.float = False
normalization.inputs.collapse_output_transforms = True
normalization.inputs.convergence_threshold = [1e-06]
normalization.inputs.convergence_window_size = [10]
normalization.inputs.dimension = 3
normalization.inputs.fixed_image = MNItemplate
normalization.inputs.initial_moving_transform_com = True
normalization.inputs.metric = ['MI', 'MI', 'CC']
normalization.inputs.metric_weight = [1.0] * 3
normalization.inputs.number_of_iterations = [[1000, 500, 250, 100],
[1000, 500, 250, 100],
[100, 70, 50, 20]]
normalization.inputs.num_threads = ants_threads
normalization.inputs.output_transform_prefix = 'anat2template'
normalization.inputs.output_inverse_warped_image = True
normalization.inputs.output_warped_image = True
normalization.inputs.radius_or_number_of_bins = [32, 32, 4]
normalization.inputs.sampling_percentage = [0.25, 0.25, 1]
normalization.inputs.sampling_strategy = ['Regular', 'Regular', 'None']
normalization.inputs.shrink_factors = [[8, 4, 2, 1]] * 3
normalization.inputs.sigma_units = ['vox'] * 3
normalization.inputs.smoothing_sigmas = [[3, 2, 1, 0]] * 3
normalization.inputs.transforms = ['Rigid', 'Affine', 'SyN']
normalization.inputs.transform_parameters = [(0.1, ), (0.1, ),
(0.1, 3.0, 0.0)]
normalization.inputs.use_histogram_matching = True
normalization.inputs.winsorize_lower_quantile = 0.005
normalization.inputs.winsorize_upper_quantile = 0.995
normalization.inputs.write_composite_transform = True
# NEW SETTINGS (need to be adjusted; specifically shink_factors and smoothing_sigmas need to be the same length)
# normalization = Node(Registration(), name='normalization')
# normalization.inputs.float = False
# normalization.inputs.collapse_output_transforms = True
# normalization.inputs.convergence_threshold = [1e-06, 1e-06, 1e-07]
# normalization.inputs.convergence_window_size = [10]
# normalization.inputs.dimension = 3
# normalization.inputs.fixed_image = MNItemplate
# normalization.inputs.initial_moving_transform_com = True
# normalization.inputs.metric = ['MI', 'MI', 'CC']
# normalization.inputs.metric_weight = [1.0]*3
# normalization.inputs.number_of_iterations = [[1000, 500, 250, 100],
# [1000, 500, 250, 100],
# [100, 70, 50, 20]]
# normalization.inputs.num_threads = ants_threads
# normalization.inputs.output_transform_prefix = 'anat2template'
# normalization.inputs.output_inverse_warped_image = True
# normalization.inputs.output_warped_image = True
# normalization.inputs.radius_or_number_of_bins = [32, 32, 4]
# normalization.inputs.sampling_percentage = [0.25, 0.25, 1]
# normalization.inputs.sampling_strategy = ['Regular',
# 'Regular',
# 'None']
# normalization.inputs.shrink_factors = [[4, 3, 2, 1]]*3
# normalization.inputs.sigma_units = ['vox']*3
# normalization.inputs.smoothing_sigmas = [[2, 1], [2, 1], [3, 2, 1, 0]]
# normalization.inputs.transforms = ['Rigid', 'Affine', 'SyN']
# normalization.inputs.transform_parameters = [(0.1,),
# (0.1,),
# (0.1, 3.0, 0.0)]
# normalization.inputs.use_histogram_matching = True
# normalization.inputs.winsorize_lower_quantile = 0.005
# normalization.inputs.winsorize_upper_quantile = 0.995
# normalization.inputs.write_composite_transform = True
###################################
### APPLY TRANSFORMS AND SMOOTH ###
###################################
merge_transforms = Node(Merge(2),
iterfield=['in2'],
name='merge_transforms')
# Used for epi -> mni, via (coreg + norm)
apply_transforms = Node(ApplyTransforms(),
iterfield=['input_image'],
name='apply_transforms')
apply_transforms.inputs.input_image_type = 3
apply_transforms.inputs.float = False
apply_transforms.inputs.num_threads = 12
apply_transforms.inputs.environ = {}
apply_transforms.inputs.interpolation = 'BSpline'
apply_transforms.inputs.invert_transform_flags = [False, False]
apply_transforms.inputs.reference_image = MNItemplate
# Used for t1 segmented -> mni, via (norm)
apply_transform_seg = Node(ApplyTransforms(), name='apply_transform_seg')
apply_transform_seg.inputs.input_image_type = 3
apply_transform_seg.inputs.float = False
apply_transform_seg.inputs.num_threads = 12
apply_transform_seg.inputs.environ = {}
apply_transform_seg.inputs.interpolation = 'MultiLabel'
apply_transform_seg.inputs.invert_transform_flags = [False]
apply_transform_seg.inputs.reference_image = MNItemplate
###################################
### PLOTS ###
###################################
plot_realign = Node(Plot_Realignment_Parameters(), name="plot_realign")
plot_qa = Node(Plot_Quality_Control(), name="plot_qa")
plot_normalization_check = Node(Plot_Coregistration_Montage(),
name="plot_normalization_check")
plot_normalization_check.inputs.canonical_img = MNItemplatehasskull
############################################
### FILTER, SMOOTH, DOWNSAMPLE PRECISION ###
############################################
# Use cosanlab_preproc for down sampling
down_samp = Node(Down_Sample_Precision(), name="down_samp")
# Use FSL for smoothing
if apply_smooth:
smooth = Node(Smooth(), name='smooth')
if isinstance(apply_smooth, list):
smooth.iterables = ("fwhm", apply_smooth)
elif isinstance(apply_smooth, int) or isinstance(apply_smooth, float):
smooth.inputs.fwhm = apply_smooth
else:
raise ValueError("apply_smooth must be a list or int/float")
# Use cosanlab_preproc for low-pass filtering
if apply_filter:
lp_filter = Node(Filter_In_Mask(), name='lp_filter')
lp_filter.inputs.mask = MNImask
lp_filter.inputs.sampling_rate = tr_length
lp_filter.inputs.high_pass_cutoff = 0
if isinstance(apply_filter, list):
lp_filter.iterables = ("low_pass_cutoff", apply_filter)
elif isinstance(apply_filter, int) or isinstance(apply_filter, float):
lp_filter.inputs.low_pass_cutoff = apply_filter
else:
raise ValueError("apply_filter must be a list or int/float")
###################
### OUTPUT NODE ###
###################
# Collect all final outputs in the output dir and get rid of file name additions
datasink = Node(DataSink(), name='datasink')
if session:
datasink.inputs.base_directory = os.path.join(output_final_dir,
subject_id)
datasink.inputs.container = 'ses-' + session
else:
datasink.inputs.base_directory = output_final_dir
datasink.inputs.container = subject_id
# Remove substitutions
data_dir_parts = data_dir.split('/')[1:]
if session:
prefix = ['_scan_'] + data_dir_parts + [subject_id] + [
'ses-' + session
] + ['func']
else:
prefix = ['_scan_'] + data_dir_parts + [subject_id] + ['func']
func_scan_names = [os.path.split(elem)[-1] for elem in funcs]
to_replace = []
for elem in func_scan_names:
bold_name = elem.split(subject_id + '_')[-1]
bold_name = bold_name.split('.nii.gz')[0]
to_replace.append(('..'.join(prefix + [elem]), bold_name))
datasink.inputs.substitutions = to_replace
#####################
### INIT WORKFLOW ###
#####################
# If we have sessions provide the full path to the subject's intermediate directory
# and only rely on workflow init to create the session container *within* that directory
# Otherwise just point to the intermediate directory and let the workflow init create the subject container within the intermediate directory
if session:
workflow = Workflow(name='ses_' + session)
workflow.base_dir = os.path.join(output_interm_dir, subId)
else:
workflow = Workflow(name=subId)
workflow.base_dir = output_interm_dir
############################
######### PART (1a) #########
# func -> discorr -> trim -> realign
# OR
# func -> trim -> realign
# OR
# func -> discorr -> realign
# OR
# func -> realign
############################
if apply_dist_corr:
workflow.connect([(encoding_file_writer, topup, [('encoding_file',
'encoding_file')]),
(encoding_file_writer, apply_topup,
[('encoding_file', 'encoding_file')]),
(merger, topup, [('merged_file', 'in_file')]),
(func_scans, apply_topup, [('scan', 'in_files')]),
(topup, apply_topup,
[('out_fieldcoef', 'in_topup_fieldcoef'),
('out_movpar', 'in_topup_movpar')])])
if apply_trim:
# Dist Corr + Trim
workflow.connect([(apply_topup, trim, [('out_corrected', 'in_file')
]),
(trim, realign_fsl, [('out_file', 'in_file')])])
else:
# Dist Corr + No Trim
workflow.connect([(apply_topup, realign_fsl, [('out_corrected',
'in_file')])])
else:
if apply_trim:
# No Dist Corr + Trim
workflow.connect([(func_scans, trim, [('scan', 'in_file')]),
(trim, realign_fsl, [('out_file', 'in_file')])])
else:
# No Dist Corr + No Trim
workflow.connect([
(func_scans, realign_fsl, [('scan', 'in_file')]),
])
############################
######### PART (1n) #########
# anat -> N4 -> bet
# OR
# anat -> bet
############################
if apply_n4:
workflow.connect([(n4_correction, brain_extraction_ants,
[('output_image', 'anatomical_image')])])
else:
brain_extraction_ants.inputs.anatomical_image = anat
##########################################
############### PART (2) #################
# realign -> coreg -> mni (via t1)
# t1 -> mni
# covariate creation
# plot creation
###########################################
workflow.connect([
(realign_fsl, plot_realign, [('par_file', 'realignment_parameters')]),
(realign_fsl, plot_qa, [('out_file', 'dat_img')]),
(realign_fsl, art, [('out_file', 'realigned_files'),
('par_file', 'realignment_parameters')]),
(realign_fsl, mean_epi, [('out_file', 'in_file')]),
(realign_fsl, make_cov, [('par_file', 'realignment_parameters')]),
(mean_epi, compute_mask, [('out_file', 'mean_volume')]),
(compute_mask, art, [('brain_mask', 'mask_file')]),
(art, make_cov, [('outlier_files', 'spike_id')]),
(art, plot_realign, [('outlier_files', 'outliers')]),
(plot_qa, make_cov, [('fd_outliers', 'fd_outliers')]),
(brain_extraction_ants, coregistration, [('BrainExtractionBrain',
'fixed_image')]),
(mean_epi, coregistration, [('out_file', 'moving_image')]),
(brain_extraction_ants, normalization, [('BrainExtractionBrain',
'moving_image')]),
(coregistration, merge_transforms, [('composite_transform', 'in2')]),
(normalization, merge_transforms, [('composite_transform', 'in1')]),
(merge_transforms, apply_transforms, [('out', 'transforms')]),
(realign_fsl, apply_transforms, [('out_file', 'input_image')]),
(apply_transforms, mean_norm_epi, [('output_image', 'in_file')]),
(normalization, apply_transform_seg, [('composite_transform',
'transforms')]),
(brain_extraction_ants, apply_transform_seg,
[('BrainExtractionSegmentation', 'input_image')]),
(mean_norm_epi, plot_normalization_check, [('out_file', 'wra_img')])
])
##################################################
################### PART (3) #####################
# epi (in mni) -> filter -> smooth -> down sample
# OR
# epi (in mni) -> filter -> down sample
# OR
# epi (in mni) -> smooth -> down sample
# OR
# epi (in mni) -> down sample
###################################################
if apply_filter:
workflow.connect([(apply_transforms, lp_filter, [('output_image',
'in_file')])])
if apply_smooth:
# Filtering + Smoothing
workflow.connect([(lp_filter, smooth, [('out_file', 'in_file')]),
(smooth, down_samp, [('smoothed_file', 'in_file')
])])
else:
# Filtering + No Smoothing
workflow.connect([(lp_filter, down_samp, [('out_file', 'in_file')])
])
else:
if apply_smooth:
# No Filtering + Smoothing
workflow.connect([
(apply_transforms, smooth, [('output_image', 'in_file')]),
(smooth, down_samp, [('smoothed_file', 'in_file')])
])
else:
# No Filtering + No Smoothing
workflow.connect([(apply_transforms, down_samp, [('output_image',
'in_file')])])
##########################################
############### PART (4) #################
# down sample -> save
# plots -> save
# covs -> save
# t1 (in mni) -> save
# t1 segmented masks (in mni) -> save
# realignment parms -> save
##########################################
workflow.connect([
(down_samp, datasink, [('out_file', 'functional.@down_samp')]),
(plot_realign, datasink, [('plot', 'functional.@plot_realign')]),
(plot_qa, datasink, [('plot', 'functional.@plot_qa')]),
(plot_normalization_check, datasink,
[('plot', 'functional.@plot_normalization')]),
(make_cov, datasink, [('covariates', 'functional.@covariates')]),
(normalization, datasink, [('warped_image', 'structural.@normanat')]),
(apply_transform_seg, datasink, [('output_image',
'structural.@normanatseg')]),
(realign_fsl, datasink, [('par_file', 'functional.@motionparams')])
])
if not os.path.exists(os.path.join(output_dir, 'pipeline.png')):
workflow.write_graph(dotfilename=os.path.join(output_dir, 'pipeline'),
format='png')
print(f"Creating workflow for subject: {subject_id}")
if ants_threads != 8:
print(
f"ANTs will utilize the user-requested {ants_threads} threads for parallel processing."
)
return workflow
def learning_predict_data_2samp_wf(working_dir,
ds_dir,
in_data_name_list,
subjects_selection_crit_dict,
subjects_selection_crit_names_list,
aggregated_subjects_dir,
target_list,
use_n_procs,
plugin_name,
confound_regression=[False, True],
run_cv=False,
n_jobs_cv=1,
run_tuning=False,
run_2sample_training=False,
aggregated_subjects_dir_nki=None,
subjects_selection_crit_dict_nki=None,
subjects_selection_crit_name_nki=None,
reverse_split=False,
random_state_nki=666,
run_learning_curve=False,
life_test_size=0.5):
import os
from nipype import config
from nipype.pipeline.engine import Node, Workflow
import nipype.interfaces.utility as util
import nipype.interfaces.io as nio
from itertools import chain
from learning_utils import aggregate_multimodal_metrics_fct, run_prediction_split_fct, \
backproject_and_split_weights_fct, select_subjects_fct, select_multimodal_X_fct, learning_curve_plot
import pandas as pd
###############################################################################################################
# GENERAL SETTINGS
wf = Workflow(name='learning_predict_data_2samp_wf')
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'),
execution={'stop_on_first_crash': False,
'remove_unnecessary_outputs': False,
'job_finished_timeout': 120})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash')
ds = Node(nio.DataSink(), name='ds')
ds.inputs.base_directory = os.path.join(ds_dir, 'group_learning_prepare_data')
ds.inputs.regexp_substitutions = [
# ('subject_id_', ''),
('_parcellation_', ''),
('_bp_freqs_', 'bp_'),
('_extraction_method_', ''),
('_subject_id_[A0-9]*/', '')
]
ds_pdf = Node(nio.DataSink(), name='ds_pdf')
ds_pdf.inputs.base_directory = os.path.join(ds_dir, 'pdfs')
ds_pdf.inputs.parameterization = False
###############################################################################################################
# ensure in_data_name_list is list of lists
in_data_name_list = [i if type(i) == list else [i] for i in in_data_name_list]
in_data_name_list_unique = list(set(chain.from_iterable(in_data_name_list)))
###############################################################################################################
# SET ITERATORS
in_data_name_infosource = Node(util.IdentityInterface(fields=['in_data_name']), name='in_data_name_infosource')
in_data_name_infosource.iterables = ('in_data_name', in_data_name_list_unique)
multimodal_in_data_name_infosource = Node(util.IdentityInterface(fields=['multimodal_in_data_name']),
name='multimodal_in_data_name_infosource')
multimodal_in_data_name_infosource.iterables = ('multimodal_in_data_name', in_data_name_list)
subject_selection_infosource = Node(util.IdentityInterface(fields=['selection_criterium']),
name='subject_selection_infosource')
subject_selection_infosource.iterables = ('selection_criterium', subjects_selection_crit_names_list)
target_infosource = Node(util.IdentityInterface(fields=['target_name']), name='target_infosource')
target_infosource.iterables = ('target_name', target_list)
###############################################################################################################
# COMPILE LIFE DATA
###############################################################################################################
###############################################################################################################
# GET INFO AND SELECT FILES
df_all_subjects_pickle_file = os.path.join(aggregated_subjects_dir, 'df_all_subjects_pickle_file/df_all.pkl')
df = pd.read_pickle(df_all_subjects_pickle_file)
# build lookup dict for unimodal data
X_file_template = 'X_file/_in_data_name_{in_data_name}/vectorized_aggregated_data.npy'
info_file_template = 'unimodal_backprojection_info_file/_in_data_name_{in_data_name}/unimodal_backprojection_info.pkl'
unimodal_lookup_dict = {}
for k in in_data_name_list_unique:
unimodal_lookup_dict[k] = {'X_file': os.path.join(aggregated_subjects_dir, X_file_template.format(
in_data_name=k)),
'unimodal_backprojection_info_file': os.path.join(aggregated_subjects_dir,
info_file_template.format(
in_data_name=k))
}
###############################################################################################################
# AGGREGATE MULTIMODAL METRICS
# stack single modality arrays horizontally
aggregate_multimodal_metrics = Node(util.Function(input_names=['multimodal_list', 'unimodal_lookup_dict'],
output_names=['X_multimodal_file',
'multimodal_backprojection_info',
'multimodal_name'],
function=aggregate_multimodal_metrics_fct),
name='aggregate_multimodal_metrics')
wf.connect(multimodal_in_data_name_infosource, 'multimodal_in_data_name', aggregate_multimodal_metrics,
'multimodal_list')
aggregate_multimodal_metrics.inputs.unimodal_lookup_dict = unimodal_lookup_dict
###############################################################################################################
# GET INDEXER FOR SUBJECTS OF INTEREST (as defined by selection criterium)
select_subjects = Node(util.Function(input_names=['df_all_subjects_pickle_file',
'subjects_selection_crit_dict',
'selection_criterium'],
output_names=['df_use_file',
'df_use_pickle_file',
'subjects_selection_index'],
function=select_subjects_fct),
name='select_subjects')
select_subjects.inputs.df_all_subjects_pickle_file = df_all_subjects_pickle_file
select_subjects.inputs.subjects_selection_crit_dict = subjects_selection_crit_dict
wf.connect(subject_selection_infosource, 'selection_criterium', select_subjects, 'selection_criterium')
###############################################################################################################
# SELECT MULITMODAL X
# select subjects (rows) from multimodal X according indexer
select_multimodal_X = Node(util.Function(input_names=['X_multimodal_file', 'subjects_selection_index',
'selection_criterium'],
output_names=['X_multimodal_selected_file'],
function=select_multimodal_X_fct),
name='select_multimodal_X')
wf.connect(aggregate_multimodal_metrics, 'X_multimodal_file', select_multimodal_X, 'X_multimodal_file')
wf.connect(select_subjects, 'subjects_selection_index', select_multimodal_X, 'subjects_selection_index')
###############################################################################################################
# COMPILE NKI DATA
###############################################################################################################
if run_2sample_training:
###############################################################################################################
# GET INFO AND SELECT FILES
df_all_subjects_pickle_file_nki = os.path.join(aggregated_subjects_dir_nki,
'df_all_subjects_pickle_file/df_all.pkl')
df_nki = pd.read_pickle(df_all_subjects_pickle_file_nki)
# build lookup dict for unimodal data
X_file_template = 'X_file/_in_data_name_{in_data_name}/vectorized_aggregated_data.npy'
info_file_template = 'unimodal_backprojection_info_file/_in_data_name_{in_data_name}/unimodal_backprojection_info.pkl'
unimodal_lookup_dict_nki = {}
for k in in_data_name_list_unique:
unimodal_lookup_dict_nki[k] = {'X_file': os.path.join(aggregated_subjects_dir_nki, X_file_template.format(
in_data_name=k)),
'unimodal_backprojection_info_file': os.path.join(
aggregated_subjects_dir_nki,
info_file_template.format(
in_data_name=k))
}
###############################################################################################################
# AGGREGATE MULTIMODAL METRICS
# stack single modality arrays horizontally
aggregate_multimodal_metrics_nki = Node(util.Function(input_names=['multimodal_list', 'unimodal_lookup_dict'],
output_names=['X_multimodal_file',
'multimodal_backprojection_info',
'multimodal_name'],
function=aggregate_multimodal_metrics_fct),
name='aggregate_multimodal_metrics_nki')
wf.connect(multimodal_in_data_name_infosource, 'multimodal_in_data_name', aggregate_multimodal_metrics_nki,
'multimodal_list')
aggregate_multimodal_metrics_nki.inputs.unimodal_lookup_dict = unimodal_lookup_dict_nki
###############################################################################################################
# GET INDEXER FOR SUBJECTS OF INTEREST (as defined by selection criterium)
select_subjects_nki = Node(util.Function(input_names=['df_all_subjects_pickle_file',
'subjects_selection_crit_dict',
'selection_criterium'],
output_names=['df_use_file',
'df_use_pickle_file',
'subjects_selection_index'],
function=select_subjects_fct),
name='select_subjects_nki')
select_subjects_nki.inputs.df_all_subjects_pickle_file = df_all_subjects_pickle_file_nki
select_subjects_nki.inputs.subjects_selection_crit_dict = subjects_selection_crit_dict_nki
select_subjects_nki.inputs.selection_criterium = subjects_selection_crit_name_nki
###############################################################################################################
# SELECT MULITMODAL X
# select subjects (rows) from multimodal X according indexer
select_multimodal_X_nki = Node(util.Function(input_names=['X_multimodal_file', 'subjects_selection_index',
'selection_criterium'],
output_names=['X_multimodal_selected_file'],
function=select_multimodal_X_fct),
name='select_multimodal_X_nki')
wf.connect(aggregate_multimodal_metrics_nki, 'X_multimodal_file', select_multimodal_X_nki, 'X_multimodal_file')
wf.connect(select_subjects_nki, 'subjects_selection_index', select_multimodal_X_nki, 'subjects_selection_index')
###############################################################################################################
# RUN PREDICTION
#
prediction_node_dict = {}
backprojection_node_dict = {}
prediction_split = Node(util.Function(input_names=['X_file',
'target_name',
'selection_criterium',
'df_file',
'data_str',
'regress_confounds',
'run_cv',
'n_jobs_cv',
'run_tuning',
'X_file_nki',
'df_file_nki',
'reverse_split',
'random_state_nki',
'run_learning_curve',
'life_test_size'],
output_names=['scatter_file',
'brain_age_scatter_file',
'df_life_out_file',
'df_nki_out_file',
'df_big_out_file',
'model_out_file',
'df_res_out_file',
'tuning_curve_file',
'scatter_file_cv',
'learning_curve_plot_file',
'learning_curve_df_file'],
function=run_prediction_split_fct),
name='prediction_split')
backproject_and_split_weights = Node(util.Function(input_names=['trained_model_file',
'multimodal_backprojection_info',
'data_str',
'target_name'],
output_names=['out_file_list',
'out_file_render_list'],
function=backproject_and_split_weights_fct),
name='backproject_and_split_weights')
i = 0
for reg in confound_regression:
the_out_node_str = 'single_source_model_reg_%s_' % (reg)
prediction_node_dict[i] = prediction_split.clone(the_out_node_str)
the_in_node = prediction_node_dict[i]
the_in_node.inputs.regress_confounds = reg
the_in_node.inputs.run_cv = run_cv
the_in_node.inputs.n_jobs_cv = n_jobs_cv
the_in_node.inputs.run_tuning = run_tuning
the_in_node.inputs.reverse_split = reverse_split
the_in_node.inputs.random_state_nki = random_state_nki
the_in_node.inputs.run_learning_curve = run_learning_curve
the_in_node.inputs.life_test_size = life_test_size
wf.connect(select_multimodal_X, 'X_multimodal_selected_file', the_in_node, 'X_file')
wf.connect(target_infosource, 'target_name', the_in_node, 'target_name')
wf.connect(subject_selection_infosource, 'selection_criterium', the_in_node, 'selection_criterium')
wf.connect(select_subjects, 'df_use_pickle_file', the_in_node, 'df_file')
wf.connect(aggregate_multimodal_metrics, 'multimodal_name', the_in_node, 'data_str')
wf.connect(the_in_node, 'model_out_file', ds, the_out_node_str + 'trained_model')
wf.connect(the_in_node, 'scatter_file', ds_pdf, the_out_node_str + 'scatter')
wf.connect(the_in_node, 'brain_age_scatter_file', ds_pdf, the_out_node_str + 'brain_age_scatter')
wf.connect(the_in_node, 'df_life_out_file', ds_pdf, the_out_node_str + 'predicted_life')
wf.connect(the_in_node, 'df_nki_out_file', ds_pdf, the_out_node_str + 'predicted_nki')
wf.connect(the_in_node, 'df_big_out_file', ds_pdf, the_out_node_str + 'predicted')
wf.connect(the_in_node, 'df_res_out_file', ds_pdf, the_out_node_str + 'results_error')
wf.connect(the_in_node, 'tuning_curve_file', ds_pdf, the_out_node_str + 'tuning_curve')
wf.connect(the_in_node, 'scatter_file_cv', ds_pdf, the_out_node_str + 'scatter_cv')
wf.connect(the_in_node, 'learning_curve_plot_file', ds_pdf, the_out_node_str + 'learning_curve_plot_file.@plot')
wf.connect(the_in_node, 'learning_curve_df_file', ds_pdf, the_out_node_str + 'learning_curve_df_file.@df')
# NKI
if run_2sample_training:
wf.connect(select_multimodal_X_nki, 'X_multimodal_selected_file', the_in_node, 'X_file_nki')
wf.connect(select_subjects_nki, 'df_use_pickle_file', the_in_node, 'df_file_nki')
else:
the_in_node.inputs.df_file_nki = None
the_in_node.inputs.X_file_nki = None
# BACKPROJECT PREDICTION WEIGHTS
# map weights back to single modality original format (e.g., nifti or matrix)
the_out_node_str = 'backprojection_single_source_model_reg_%s_' % (reg)
backprojection_node_dict[i] = backproject_and_split_weights.clone(the_out_node_str)
the_from_node = prediction_node_dict[i]
the_in_node = backprojection_node_dict[i]
wf.connect(the_from_node, 'model_out_file', the_in_node, 'trained_model_file')
wf.connect(aggregate_multimodal_metrics, 'multimodal_backprojection_info', the_in_node,
'multimodal_backprojection_info')
wf.connect(aggregate_multimodal_metrics, 'multimodal_name', the_in_node, 'data_str')
wf.connect(target_infosource, 'target_name', the_in_node, 'target_name')
wf.connect(the_in_node, 'out_file_list', ds_pdf, the_out_node_str + '.@weights')
wf.connect(the_in_node, 'out_file_render_list', ds_pdf, the_out_node_str + 'renders.@renders')
i += 1
###############################################################################################################
# # RUN WF
wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical')
wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf')
wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf')
if plugin_name == 'CondorDAGMan':
wf.run(plugin=plugin_name)
if plugin_name == 'MultiProc':
wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def process(self):
# Enable the use of the the W3C PROV data model to capture and represent provenance in Nipype
# config.enable_provenance()
# Process time
self.now = datetime.datetime.now().strftime("%Y%m%d_%H%M")
if '_' in self.subject:
self.subject = self.subject.split('_')[0]
# old_subject = self.subject
if self.global_conf.subject_session == '':
cmp_deriv_subject_directory = os.path.join(self.output_directory,
"cmp", self.subject)
nipype_deriv_subject_directory = os.path.join(
self.output_directory, "nipype", self.subject)
else:
cmp_deriv_subject_directory = os.path.join(
self.output_directory, "cmp", self.subject,
self.global_conf.subject_session)
nipype_deriv_subject_directory = os.path.join(
self.output_directory, "nipype", self.subject,
self.global_conf.subject_session)
self.subject = "_".join(
(self.subject, self.global_conf.subject_session))
if not os.path.exists(
os.path.join(nipype_deriv_subject_directory, "fMRI_pipeline")):
try:
os.makedirs(
os.path.join(nipype_deriv_subject_directory,
"fMRI_pipeline"))
except os.error:
print("%s was already existing" % os.path.join(
nipype_deriv_subject_directory, "fMRI_pipeline"))
# Initialization
if os.path.isfile(
os.path.join(nipype_deriv_subject_directory, "fMRI_pipeline",
"pypeline.log")):
os.unlink(
os.path.join(nipype_deriv_subject_directory, "fMRI_pipeline",
"pypeline.log"))
config.update_config({
'logging': {
'log_directory':
os.path.join(nipype_deriv_subject_directory, "fMRI_pipeline"),
'log_to_file':
True
},
'execution': {
'remove_unnecessary_outputs': False,
'stop_on_first_crash': True,
'stop_on_first_rerun': False,
'use_relative_paths': True,
'crashfile_format': "txt"
}
})
logging.update_logging(config)
iflogger = logging.getLogger('nipype.interface')
iflogger.info("**** Processing ****")
flow = self.create_pipeline_flow(
cmp_deriv_subject_directory=cmp_deriv_subject_directory,
nipype_deriv_subject_directory=nipype_deriv_subject_directory)
flow.write_graph(graph2use='colored', format='svg', simple_form=False)
# try:
if (self.number_of_cores != 1):
flow.run(plugin='MultiProc',
plugin_args={'n_procs': self.number_of_cores})
else:
flow.run()
# self.fill_stages_outputs()
iflogger.info("**** Processing finished ****")
return True, 'Processing successful'
if __name__ == '__main__':
import sys
import datetime, time
from nipype import config
from dmri_pipe1_prepro import do_pipe1_prepro
from dmri_pipe2_tractscript import script_tracking, trackwait
from dmri_pipe3_projection import do_pipe3_projection
from dmri_pipe4_distmat import do_pipe4_distmat
from dmri_pipe5_distmat_lr import do_pipe5_distmat_lr
from dmri_pipe_cleanup import do_cleanup, do_wrapup
cfg = dict(logging=dict(workflow_level = 'INFO'), execution={'remove_unnecessary_outputs': False, 'job_finished_timeout': 120, 'stop_on_first_rerun': False, 'stop_on_first_crash': True} )
config.update_config(cfg)
tract_number = 5000
tract_step = 0.3
freesurfer_dir = '/scr/kalifornien1/data/nki_enhanced/freesurfer'
data_dir = '/scr/kalifornien1/data/nki_enhanced/dicoms/diff_2_nii'
# freesurfer_dir = '/scr/kongo2/NKI_ENH/freesurfer'
# data_dir = '/scr/kongo2/NKI_ENH/dMRI'
data_template = "%s/DTI_mx_137/%s"
is_LH = True
is_RH = False
pipe_dict = dict([('0',0),('1',0),('2',1),('21',1),('22',2),('3',3),('4',4),('41',4),('42',5),('5',6)])
pipe_stop = 7
def segmentation(projectid, subjectid, sessionid, master_config, onlyT1=True, pipeline_name=''):
import os.path
import nipype.pipeline.engine as pe
import nipype.interfaces.io as nio
from nipype.interfaces import ants
from nipype.interfaces.utility import IdentityInterface, Function, Merge
# Set universal pipeline options
from nipype import config
config.update_config(master_config)
assert config.get('execution', 'plugin') == master_config['execution']['plugin']
from PipeLineFunctionHelpers import ClipT1ImageWithBrainMask
from WorkupT1T2BRAINSCut import CreateBRAINSCutWorkflow
from utilities.distributed import modify_qsub_args
from SEMTools import BRAINSSnapShotWriter
baw200 = pe.Workflow(name=pipeline_name)
# HACK: print for debugging
for key, itme in master_config.items():
print "-" * 30
print key, ":", itme
print "-" * 30
#END HACK
inputsSpec = pe.Node(interface=IdentityInterface(fields=['t1_average',
't2_average',
'template_t1',
'hncma-atlas',
'LMIatlasToSubject_tx',
'inputLabels',
'inputHeadLabels',
'posteriorImages',
'TissueClassifyatlasToSubjectInverseTransform',
'UpdatedPosteriorsList']),
run_without_submitting=True, name='inputspec')
# outputsSpec = pe.Node(interface=IdentityInterface(fields=[...]),
# run_without_submitting=True, name='outputspec')
currentClipT1ImageWithBrainMaskName = 'ClipT1ImageWithBrainMask_' + str(subjectid) + "_" + str(sessionid)
ClipT1ImageWithBrainMaskNode = pe.Node(interface=Function(function=ClipT1ImageWithBrainMask,
input_names=['t1_image', 'brain_labels',
'clipped_file_name'],
output_names=['clipped_file']),
name=currentClipT1ImageWithBrainMaskName)
ClipT1ImageWithBrainMaskNode.inputs.clipped_file_name = 'clipped_from_BABC_labels_t1.nii.gz'
baw200.connect([(inputsSpec, ClipT1ImageWithBrainMaskNode, [('t1_average', 't1_image'),
('inputLabels', 'brain_labels')])])
currentAtlasToSubjectantsRegistration = 'AtlasToSubjectANTsRegistration_' + str(subjectid) + "_" + str(sessionid)
AtlasToSubjectantsRegistration = pe.Node(interface=ants.Registration(), name=currentAtlasToSubjectantsRegistration)
AtlasToSubjectantsRegistration.inputs.dimension = 3
AtlasToSubjectantsRegistration.inputs.transforms = ["Affine", "SyN"]
AtlasToSubjectantsRegistration.inputs.transform_parameters = [[0.1], [0.15, 3.0, 0.0]]
AtlasToSubjectantsRegistration.inputs.metric = ['Mattes', 'CC']
AtlasToSubjectantsRegistration.inputs.sampling_strategy = ['Regular', None]
AtlasToSubjectantsRegistration.inputs.sampling_percentage = [1.0, 1.0]
AtlasToSubjectantsRegistration.inputs.metric_weight = [1.0, 1.0]
AtlasToSubjectantsRegistration.inputs.radius_or_number_of_bins = [32, 4]
AtlasToSubjectantsRegistration.inputs.number_of_iterations = [[1000, 1000, 1000], [10000, 500, 500, 200]]
AtlasToSubjectantsRegistration.inputs.convergence_threshold = [5e-7, 5e-7]
AtlasToSubjectantsRegistration.inputs.convergence_window_size = [25, 25]
AtlasToSubjectantsRegistration.inputs.use_histogram_matching = [True, True]
AtlasToSubjectantsRegistration.inputs.shrink_factors = [[4, 2, 1], [5, 4, 2, 1]]
AtlasToSubjectantsRegistration.inputs.smoothing_sigmas = [[4, 2, 0], [5, 4, 2, 0]]
AtlasToSubjectantsRegistration.inputs.sigma_units = ["vox","vox"]
AtlasToSubjectantsRegistration.inputs.use_estimate_learning_rate_once = [False, False]
AtlasToSubjectantsRegistration.inputs.write_composite_transform = True
AtlasToSubjectantsRegistration.inputs.collapse_output_transforms = True
AtlasToSubjectantsRegistration.inputs.output_transform_prefix = 'AtlasToSubject_'
AtlasToSubjectantsRegistration.inputs.winsorize_lower_quantile = 0.025
AtlasToSubjectantsRegistration.inputs.winsorize_upper_quantile = 0.975
AtlasToSubjectantsRegistration.inputs.collapse_linear_transforms_to_fixed_image_header = False
AtlasToSubjectantsRegistration.inputs.output_warped_image = 'atlas2subject.nii.gz'
AtlasToSubjectantsRegistration.inputs.output_inverse_warped_image = 'subject2atlas.nii.gz'
baw200.connect([(inputsSpec, AtlasToSubjectantsRegistration, [('LMIatlasToSubject_tx', 'initial_moving_transform'),
('t1_average', 'fixed_image'),
('template_t1', 'moving_image')])
])
myLocalSegWF = CreateBRAINSCutWorkflow(projectid,
subjectid,
sessionid,
master_config['queue'],
master_config['long_q'],
t1Only=onlyT1)
MergeStage2AverageImagesName = "99_mergeAvergeStage2Images_" + str(sessionid)
MergeStage2AverageImages = pe.Node(interface=Merge(2), run_without_submitting=True,
name=MergeStage2AverageImagesName)
baw200.connect([(inputsSpec, myLocalSegWF, [('t1_average', 'inputspec.T1Volume'),
('posteriorImages', "inputspec.posteriorDictionary"),
('inputLabels', 'inputspec.RegistrationROI'),]),
(inputsSpec, MergeStage2AverageImages, [('t1_average', 'in1')]),
(AtlasToSubjectantsRegistration, myLocalSegWF, [('composite_transform',
'inputspec.atlasToSubjectTransform')])
])
if not onlyT1:
baw200.connect([(inputsSpec, myLocalSegWF, [('t2_average', 'inputspec.T2Volume')]),
(inputsSpec, MergeStage2AverageImages, [('t2_average', 'in2')])])
file_count = 15 # Count of files to merge into MergeSessionSubjectToAtlas
else:
file_count = 14 # Count of files to merge into MergeSessionSubjectToAtlas
## NOTE: Element 0 of AccumulatePriorsList is the accumulated GM tissue
# baw200.connect([(AccumulateLikeTissuePosteriorsNode, myLocalSegWF,
# [(('AccumulatePriorsList', getListIndex, 0), "inputspec.TotalGM")]),
# ])
### Now define where the final organized outputs should go.
DataSink = pe.Node(nio.DataSink(), name="CleanedDenoisedSegmentation_DS_" + str(subjectid) + "_" + str(sessionid))
DataSink.overwrite = master_config['ds_overwrite']
DataSink.inputs.base_directory = master_config['resultdir']
# DataSink.inputs.regexp_substitutions = GenerateOutputPattern(projectid, subjectid, sessionid,'BRAINSCut')
# DataSink.inputs.regexp_substitutions = GenerateBRAINSCutImagesOutputPattern(projectid, subjectid, sessionid)
DataSink.inputs.substitutions = [('Segmentations', os.path.join(projectid, subjectid, sessionid, 'CleanedDenoisedRFSegmentations')),
('subjectANNLabel_', ''),
('ANNContinuousPrediction', ''),
('subject.nii.gz', '.nii.gz'),
('_seg.nii.gz', '_seg.nii.gz'),
('.nii.gz', '_seg.nii.gz'),
('_seg_seg', '_seg')]
baw200.connect([(myLocalSegWF, DataSink, [('outputspec.outputBinaryLeftCaudate', 'Segmentations.@LeftCaudate'),
('outputspec.outputBinaryRightCaudate', 'Segmentations.@RightCaudate'),
('outputspec.outputBinaryLeftHippocampus', 'Segmentations.@LeftHippocampus'),
('outputspec.outputBinaryRightHippocampus', 'Segmentations.@RightHippocampus'),
('outputspec.outputBinaryLeftPutamen', 'Segmentations.@LeftPutamen'),
('outputspec.outputBinaryRightPutamen', 'Segmentations.@RightPutamen'),
('outputspec.outputBinaryLeftThalamus', 'Segmentations.@LeftThalamus'),
('outputspec.outputBinaryRightThalamus', 'Segmentations.@RightThalamus'),
('outputspec.outputBinaryLeftAccumben', 'Segmentations.@LeftAccumben'),
('outputspec.outputBinaryRightAccumben', 'Segmentations.@RightAccumben'),
('outputspec.outputBinaryLeftGlobus', 'Segmentations.@LeftGlobus'),
('outputspec.outputBinaryRightGlobus', 'Segmentations.@RightGlobus'),
('outputspec.outputLabelImageName', 'Segmentations.@LabelImageName'),
('outputspec.outputCSVFileName', 'Segmentations.@CSVFileName')]),
# (myLocalSegWF, DataSink, [('outputspec.cleaned_labels', 'Segmentations.@cleaned_labels')])
])
MergeStage2BinaryVolumesName = "99_MergeStage2BinaryVolumes_" + str(sessionid)
MergeStage2BinaryVolumes = pe.Node(interface=Merge(12), run_without_submitting=True,
name=MergeStage2BinaryVolumesName)
baw200.connect([(myLocalSegWF, MergeStage2BinaryVolumes, [('outputspec.outputBinaryLeftAccumben', 'in1'),
('outputspec.outputBinaryLeftCaudate', 'in2'),
('outputspec.outputBinaryLeftPutamen', 'in3'),
('outputspec.outputBinaryLeftGlobus', 'in4'),
('outputspec.outputBinaryLeftThalamus', 'in5'),
('outputspec.outputBinaryLeftHippocampus', 'in6'),
('outputspec.outputBinaryRightAccumben', 'in7'),
('outputspec.outputBinaryRightCaudate', 'in8'),
('outputspec.outputBinaryRightPutamen', 'in9'),
('outputspec.outputBinaryRightGlobus', 'in10'),
('outputspec.outputBinaryRightThalamus', 'in11'),
('outputspec.outputBinaryRightHippocampus', 'in12')])
])
## SnapShotWriter for Segmented result checking:
SnapShotWriterNodeName = "SnapShotWriter_" + str(sessionid)
SnapShotWriter = pe.Node(interface=BRAINSSnapShotWriter(), name=SnapShotWriterNodeName)
SnapShotWriter.inputs.outputFilename = 'snapShot' + str(sessionid) + '.png' # output specification
SnapShotWriter.inputs.inputPlaneDirection = [2, 1, 1, 1, 1, 0, 0]
SnapShotWriter.inputs.inputSliceToExtractInPhysicalPoint = [-3, -7, -3, 5, 7, 22, -22]
baw200.connect([(MergeStage2AverageImages, SnapShotWriter, [('out', 'inputVolumes')]),
(MergeStage2BinaryVolumes, SnapShotWriter, [('out', 'inputBinaryVolumes')]),
(SnapShotWriter, DataSink, [('outputFilename', 'Segmentations.@outputSnapShot')])
])
currentAntsLabelWarpToSubject = 'AntsLabelWarpToSubject' + str(subjectid) + "_" + str(sessionid)
AntsLabelWarpToSubject = pe.Node(interface=ants.ApplyTransforms(), name=currentAntsLabelWarpToSubject)
AntsLabelWarpToSubject.inputs.dimension = 3
AntsLabelWarpToSubject.inputs.output_image = 'warped_hncma_atlas_seg.nii.gz'
AntsLabelWarpToSubject.inputs.interpolation = "MultiLabel"
baw200.connect([(AtlasToSubjectantsRegistration, AntsLabelWarpToSubject, [('composite_transform', 'transforms')]),
(inputsSpec, AntsLabelWarpToSubject, [('t1_average', 'reference_image'),
('hncma-atlas', 'input_image')])
])
#####
### Now define where the final organized outputs should go.
AntsLabelWarpedToSubject_DSName = "AntsLabelWarpedToSubject_DS_" + str(sessionid)
AntsLabelWarpedToSubject_DS = pe.Node(nio.DataSink(), name=AntsLabelWarpedToSubject_DSName)
AntsLabelWarpedToSubject_DS.overwrite = master_config['ds_overwrite']
AntsLabelWarpedToSubject_DS.inputs.base_directory = master_config['resultdir']
AntsLabelWarpedToSubject_DS.inputs.substitutions = [('AntsLabelWarpedToSubject', os.path.join(projectid, subjectid, sessionid, 'AntsLabelWarpedToSubject'))]
baw200.connect([(AntsLabelWarpToSubject, AntsLabelWarpedToSubject_DS, [('output_image', 'AntsLabelWarpedToSubject')])])
MergeSessionSubjectToAtlasName = "99_MergeSessionSubjectToAtlas_" + str(sessionid)
MergeSessionSubjectToAtlas = pe.Node(interface=Merge(file_count), run_without_submitting=True,
name=MergeSessionSubjectToAtlasName)
baw200.connect([(myLocalSegWF, MergeSessionSubjectToAtlas, [('outputspec.outputBinaryLeftAccumben', 'in1'),
('outputspec.outputBinaryLeftCaudate', 'in2'),
('outputspec.outputBinaryLeftPutamen', 'in3'),
('outputspec.outputBinaryLeftGlobus', 'in4'),
('outputspec.outputBinaryLeftThalamus', 'in5'),
('outputspec.outputBinaryLeftHippocampus', 'in6'),
('outputspec.outputBinaryRightAccumben', 'in7'),
('outputspec.outputBinaryRightCaudate', 'in8'),
('outputspec.outputBinaryRightPutamen', 'in9'),
('outputspec.outputBinaryRightGlobus', 'in10'),
('outputspec.outputBinaryRightThalamus', 'in11'),
('outputspec.outputBinaryRightHippocampus', 'in12')]),
# (FixWMPartitioningNode, MergeSessionSubjectToAtlas, [('UpdatedPosteriorsList', 'in13')]),
(inputsSpec, MergeSessionSubjectToAtlas, [('UpdatedPosteriorsList', 'in13')]),
(inputsSpec, MergeSessionSubjectToAtlas, [('t1_average', 'in14')])
])
if not onlyT1:
assert file_count == 15
baw200.connect([(inputsSpec, MergeSessionSubjectToAtlas, [('t2_average', 'in15')])])
LinearSubjectToAtlasANTsApplyTransformsName = 'LinearSubjectToAtlasANTsApplyTransforms_' + str(sessionid)
LinearSubjectToAtlasANTsApplyTransforms = pe.MapNode(interface=ants.ApplyTransforms(), iterfield=['input_image'],
name=LinearSubjectToAtlasANTsApplyTransformsName)
LinearSubjectToAtlasANTsApplyTransforms.inputs.interpolation = 'Linear'
baw200.connect([(AtlasToSubjectantsRegistration, LinearSubjectToAtlasANTsApplyTransforms, [('inverse_composite_transform',
'transforms')]),
(inputsSpec, LinearSubjectToAtlasANTsApplyTransforms, [('template_t1', 'reference_image')]),
(MergeSessionSubjectToAtlas, LinearSubjectToAtlasANTsApplyTransforms, [('out', 'input_image')])
])
MergeMultiLabelSessionSubjectToAtlasName = "99_MergeMultiLabelSessionSubjectToAtlas_" + str(sessionid)
MergeMultiLabelSessionSubjectToAtlas = pe.Node(interface=Merge(2), run_without_submitting=True,
name=MergeMultiLabelSessionSubjectToAtlasName)
baw200.connect([(inputsSpec, MergeMultiLabelSessionSubjectToAtlas, [('inputLabels', 'in1'),
('inputHeadLabels', 'in2')])
])
### This is taking this sessions RF label map back into NAC atlas space.
#{
MultiLabelSubjectToAtlasANTsApplyTransformsName = 'MultiLabelSubjectToAtlasANTsApplyTransforms_' + str(sessionid) + '_map'
MultiLabelSubjectToAtlasANTsApplyTransforms = pe.MapNode(interface=ants.ApplyTransforms(), iterfield=['input_image'],
name=MultiLabelSubjectToAtlasANTsApplyTransformsName)
MultiLabelSubjectToAtlasANTsApplyTransforms.inputs.interpolation = 'MultiLabel'
baw200.connect([(AtlasToSubjectantsRegistration, MultiLabelSubjectToAtlasANTsApplyTransforms,
[('inverse_composite_transform', 'transforms')]),
(inputsSpec, MultiLabelSubjectToAtlasANTsApplyTransforms, [('template_t1', 'reference_image')]),
(MergeMultiLabelSessionSubjectToAtlas, MultiLabelSubjectToAtlasANTsApplyTransforms,
[('out', 'input_image')])
])
#}
### Now we must take the sessions to THIS SUBJECTS personalized atlas.
#{
#}
### Now define where the final organized outputs should go.
Subj2Atlas_DSName = "SubjectToAtlas_DS_" + str(sessionid)
Subj2Atlas_DS = pe.Node(nio.DataSink(), name=Subj2Atlas_DSName)
Subj2Atlas_DS.overwrite = master_config['ds_overwrite']
Subj2Atlas_DS.inputs.base_directory = master_config['resultdir']
Subj2Atlas_DS.inputs.regexp_substitutions = [(r'_LinearSubjectToAtlasANTsApplyTransforms_[^/]*',
r'' + sessionid + '/')]
baw200.connect([(LinearSubjectToAtlasANTsApplyTransforms, Subj2Atlas_DS,
[('output_image', 'SubjectToAtlasWarped.@linear_output_images')])])
Subj2AtlasTransforms_DSName = "SubjectToAtlasTransforms_DS_" + str(sessionid)
Subj2AtlasTransforms_DS = pe.Node(nio.DataSink(), name=Subj2AtlasTransforms_DSName)
Subj2AtlasTransforms_DS.overwrite = master_config['ds_overwrite']
Subj2AtlasTransforms_DS.inputs.base_directory = master_config['resultdir']
Subj2AtlasTransforms_DS.inputs.regexp_substitutions = [(r'SubjectToAtlasWarped',
r'SubjectToAtlasWarped/' + sessionid + '/')]
baw200.connect([(AtlasToSubjectantsRegistration, Subj2AtlasTransforms_DS,
[('composite_transform', 'SubjectToAtlasWarped.@composite_transform'),
('inverse_composite_transform', 'SubjectToAtlasWarped.@inverse_composite_transform')])])
# baw200.connect([(MultiLabelSubjectToAtlasANTsApplyTransforms, Subj2Atlas_DS, [('output_image', 'SubjectToAtlasWarped.@multilabel_output_images')])])
if master_config['execution']['plugin'] == 'SGE': # for some nodes, the qsub call needs to be modified on the cluster
AtlasToSubjectantsRegistration.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'], '9000M', 4,
hard=False)}
SnapShotWriter.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'], '1000M', 1, 1, hard=False)}
LinearSubjectToAtlasANTsApplyTransforms.plugin_args = {'template': master_config['plugin_args']['template'],
'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'], '1000M',
1, hard=True)}
MultiLabelSubjectToAtlasANTsApplyTransforms.plugin_args = {'template': master_config['plugin_args']['template'],
'overwrite': True,
'qsub_args': modify_qsub_args(master_config['queue'], '1000M',
1, hard=True)}
return baw200
wf.config['execution'] = {'hash_method': 'timestamp', 'crashdump_dir': os.path.abspath(c.crashLogDirectory)}
log_dir = os.path.join(c.outputDirectory, 'logs', 'group_analysis', resource, 'model_%s' % (os.path.basename(model)))
try:
os.makedirs(log_dir)
except:
print "log_dir already exist"
# enable logging
from nipype import config
from nipype import logging
config.update_config({'logging': {'log_directory': log_dir,
'log_to_file': True}})
# Temporarily disable until solved
#logging.update_logging(config)
iflogger = logging.getLogger('interface')
group_sublist = open(subject_list, 'r')
#print >>diag, "> Opened subject list: ", subject_list
#print >>diag, ""
sublist_items = group_sublist.readlines()
input_subject_list = [line.rstrip('\n') for line in sublist_items \
def RunSubjectWorkflow(args):
"""
.-----------.
--- | Session 1 | ---> /project/subjectA/session1/phase/
/ *-----------*
.-----------. /
| Subject A | <
*-----------* \
\ .-----------.
--- | Session 2 | ---> /project/subjectA/session2/phase/
*-----------*
**** Replaces WorkflowT1T2.py ****
"""
database, start_time, subject, master_config = args
assert 'baseline' in master_config['components'] or 'longitudinal' in master_config['components'], "Baseline or Longitudinal is not in WORKFLOW_COMPONENTS!"
# HACK:
# To avoid a "sqlite3.ProgrammingError: Base Cursor.__init__ not called" error
# using multiprocessing.map_async(), re-instantiate database
# database.__init__(defaultDBName=database.dbName, subject_list=database.subjectList)
#
# END HACK
import time
from nipype import config, logging
config.update_config(master_config) # Set universal pipeline options
assert config.get('execution', 'plugin') == master_config['execution']['plugin']
# DEBUG
# config.enable_debug_mode()
# config.set('execution', 'stop_on_first_rerun', 'true')
# END DEBUG
logging.update_logging(config)
import nipype.pipeline.engine as pe
import nipype.interfaces.base as nbase
import nipype.interfaces.io as nio
from nipype.interfaces.utility import IdentityInterface, Function
import traits
from baw_exp import OpenSubjectDatabase
from SessionDB import SessionDB
from PipeLineFunctionHelpers import convertToList
from atlasNode import MakeAtlasNode
from utilities.misc import GenerateSubjectOutputPattern as outputPattern
from utilities.misc import GenerateWFName
while time.time() < start_time:
time.sleep(start_time - time.time() + 1)
print "Delaying start for {subject}".format(subject=subject)
print("===================== SUBJECT: {0} ===========================".format(subject))
subjectWorkflow = pe.Workflow(name="BAW_StandardWorkup_subject_{0}".format(subject))
subjectWorkflow.base_dir = config.get('logging', 'log_directory')
# subjectWorkflow.config['execution']['plugin'] = 'Linear' # Hardcodeded in WorkupT1T2.py - why?
# DEBUG
# subjectWorkflow.config['execution']['stop_on_first_rerun'] = 'true'
# END DEBUG
atlasNode = MakeAtlasNode(master_config['atlascache'], 'BAtlas')
sessionWorkflow = dict()
inputsSpec = dict()
sessions = database.getSessionsFromSubject(subject)
# print "These are the sessions: ", sessions
if 'baseline' in master_config['components']:
current_phase = 'baseline'
from baseline import create_baseline as create_wkfl
elif 'longitudinal' in master_config['components']:
current_phase = 'longitudinal'
from longitudinal import create_longitudial as create_wkfl
for session in sessions: # TODO (future): Replace with iterable inputSpec node and add Function node for getAllFiles()
project = database.getProjFromSession(session)
pname = "{0}_{1}".format(session, current_phase) # Long node names make graphs a pain to read/print
# pname = GenerateWFName(project, subject, session, current_phase)
print "Building session pipeline for {0}".format(session)
inputsSpec[session] = pe.Node(name='inputspec_{0}'.format(session),
interface=IdentityInterface(fields=['T1s', 'T2s', 'PDs', 'FLs', 'OTs']))
inputsSpec[session].inputs.T1s = database.getFilenamesByScantype(session, ['T1-15', 'T1-30'])
inputsSpec[session].inputs.T2s = database.getFilenamesByScantype(session, ['T2-15', 'T2-30'])
inputsSpec[session].inputs.PDs = database.getFilenamesByScantype(session, ['PD-15', 'PD-30'])
inputsSpec[session].inputs.FLs = database.getFilenamesByScantype(session, ['FL-15', 'FL-30'])
inputsSpec[session].inputs.OTs = database.getFilenamesByScantype(session, ['OTHER-15', 'OTHER-30'])
sessionWorkflow[session] = create_wkfl(project, subject, session, master_config,
interpMode='Linear', pipeline_name=pname)
subjectWorkflow.connect([(inputsSpec[session], sessionWorkflow[session], [('T1s', 'inputspec.T1s'),
('T2s', 'inputspec.T2s'),
('PDs', 'inputspec.PDs'),
('FLs', 'inputspec.FLs'),
('OTs', 'inputspec.OTHERs'),
]),
(atlasNode, sessionWorkflow[session], [('template_landmarks_50Lmks_fcsv',
'inputspec.atlasLandmarkFilename'),
('template_weights_50Lmks_wts',
'inputspec.atlasWeightFilename'),
('LLSModel_50Lmks_hdf5', 'inputspec.LLSModel'),
('T1_50Lmks_mdl', 'inputspec.inputTemplateModel')]),
])
if current_phase == 'baseline':
subjectWorkflow.connect([(atlasNode, sessionWorkflow[session], [('template_t1', 'inputspec.template_t1'),
('ExtendedAtlasDefinition_xml',
'inputspec.atlasDefinition')]),
])
else:
assert current_phase == 'longitudinal', "Phase value is unknown: {0}".format(current_phase)
from utils import run_workflow, print_workflow
if False:
print_workflow(template, plugin=master_config['execution']['plugin'], dotfilename='template')
return run_workflow(template, plugin=master_config['execution']['plugin'], plugin_args=master_config['plugin_args'])
def build_workflow(opts, retval):
"""
Create the Nipype Workflow that supports the whole execution
graph, given the inputs.
All the checks and the construction of the workflow are done
inside this function that has pickleable inputs and output
dictionary (``retval``) to allow isolation using a
``multiprocessing.Process`` that allows fmriprep to enforce
a hard-limited memory-scope.
"""
from subprocess import check_call, CalledProcessError, TimeoutExpired
from pkg_resources import resource_filename as pkgrf
from shutil import copyfile
from nipype import logging, config as ncfg
from niworkflows.utils.bids import collect_participants
from ..__about__ import __version__
from ..workflows.base import init_fmriprep_wf
from ..viz.reports import generate_reports
logger = logging.getLogger('nipype.workflow')
INIT_MSG = """
Running fMRIPREP version {version}:
* BIDS dataset path: {bids_dir}.
* Participant list: {subject_list}.
* Run identifier: {uuid}.
""".format
output_spaces = opts.output_space or []
# Validity of some inputs
# ERROR check if use_aroma was specified, but the correct template was not
if opts.use_aroma and (opts.template != 'MNI152NLin2009cAsym'
or 'template' not in output_spaces):
output_spaces.append('template')
logger.warning(
'Option "--use-aroma" requires functional images to be resampled to MNI space. '
'The argument "template" has been automatically added to the list of output '
'spaces (option "--output-space").')
if opts.cifti_output and (opts.template != 'MNI152NLin2009cAsym'
or 'template' not in output_spaces):
output_spaces.append('template')
logger.warning(
'Option "--cifti-output" requires functional images to be resampled to MNI space. '
'The argument "template" has been automatically added to the list of output '
'spaces (option "--output-space").')
# Check output_space
if 'template' not in output_spaces and (opts.use_syn_sdc
or opts.force_syn):
msg = [
'SyN SDC correction requires T1 to MNI registration, but '
'"template" is not specified in "--output-space" arguments.',
'Option --use-syn will be cowardly dismissed.'
]
if opts.force_syn:
output_spaces.append('template')
msg[1] = (
' Since --force-syn has been requested, "template" has been added to'
' the "--output-space" list.')
logger.warning(' '.join(msg))
# Set up some instrumental utilities
run_uuid = '%s_%s' % (strftime('%Y%m%d-%H%M%S'), uuid.uuid4())
# First check that bids_dir looks like a BIDS folder
bids_dir = os.path.abspath(opts.bids_dir)
subject_list = collect_participants(
bids_dir, participant_label=opts.participant_label)
# Load base plugin_settings from file if --use-plugin
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as f:
plugin_settings = loadyml(f)
plugin_settings.setdefault('plugin_args', {})
else:
# Defaults
plugin_settings = {
'plugin': 'MultiProc',
'plugin_args': {
'raise_insufficient': False,
'maxtasksperchild': 1,
}
}
# Resource management options
# Note that we're making strong assumptions about valid plugin args
# This may need to be revisited if people try to use batch plugins
nthreads = plugin_settings['plugin_args'].get('n_procs')
# Permit overriding plugin config with specific CLI options
if nthreads is None or opts.nthreads is not None:
nthreads = opts.nthreads
if nthreads is None or nthreads < 1:
nthreads = cpu_count()
plugin_settings['plugin_args']['n_procs'] = nthreads
if opts.mem_mb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_mb / 1024
omp_nthreads = opts.omp_nthreads
if omp_nthreads == 0:
omp_nthreads = min(nthreads - 1 if nthreads > 1 else cpu_count(), 8)
if 1 < nthreads < omp_nthreads:
logger.warning(
'Per-process threads (--omp-nthreads=%d) exceed total '
'threads (--nthreads/--n_cpus=%d)', omp_nthreads, nthreads)
# Set up directories
output_dir = op.abspath(opts.output_dir)
log_dir = op.join(output_dir, 'fmriprep', 'logs')
work_dir = op.abspath(opts.work_dir or 'work') # Set work/ as default
# Check and create output and working directories
os.makedirs(output_dir, exist_ok=True)
os.makedirs(log_dir, exist_ok=True)
os.makedirs(work_dir, exist_ok=True)
# Nipype config (logs and execution)
ncfg.update_config({
'logging': {
'log_directory': log_dir,
'log_to_file': True
},
'execution': {
'crashdump_dir':
log_dir,
'crashfile_format':
'txt',
'get_linked_libs':
False,
'stop_on_first_crash':
opts.stop_on_first_crash or opts.work_dir is None,
},
'monitoring': {
'enabled': opts.resource_monitor,
'sample_frequency': '0.5',
'summary_append': True,
}
})
if opts.resource_monitor:
ncfg.enable_resource_monitor()
retval['return_code'] = 0
retval['plugin_settings'] = plugin_settings
retval['bids_dir'] = bids_dir
retval['output_dir'] = output_dir
retval['work_dir'] = work_dir
retval['subject_list'] = subject_list
retval['run_uuid'] = run_uuid
retval['workflow'] = None
# Called with reports only
if opts.reports_only:
logger.log(25, 'Running --reports-only on participants %s',
', '.join(subject_list))
if opts.run_uuid is not None:
run_uuid = opts.run_uuid
retval['return_code'] = generate_reports(subject_list, output_dir,
work_dir, run_uuid)
return retval
# Build main workflow
logger.log(
25,
INIT_MSG(version=__version__,
bids_dir=bids_dir,
subject_list=subject_list,
uuid=run_uuid))
template_out_grid = opts.template_resampling_grid
if opts.output_grid_reference is not None:
logger.warning(
'Option --output-grid-reference is deprecated, please use '
'--template-resampling-grid')
template_out_grid = template_out_grid or opts.output_grid_reference
if opts.debug:
logger.warning('Option --debug is deprecated and has no effect')
retval['workflow'] = init_fmriprep_wf(
subject_list=subject_list,
task_id=opts.task_id,
echo_idx=opts.echo_idx,
run_uuid=run_uuid,
ignore=opts.ignore,
debug=opts.sloppy,
low_mem=opts.low_mem,
anat_only=opts.anat_only,
longitudinal=opts.longitudinal,
t2s_coreg=opts.t2s_coreg,
omp_nthreads=omp_nthreads,
skull_strip_template=opts.skull_strip_template,
skull_strip_fixed_seed=opts.skull_strip_fixed_seed,
work_dir=work_dir,
output_dir=output_dir,
bids_dir=bids_dir,
freesurfer=opts.run_reconall,
output_spaces=output_spaces,
template=opts.template,
medial_surface_nan=opts.medial_surface_nan,
cifti_output=opts.cifti_output,
template_out_grid=template_out_grid,
hires=opts.hires,
use_bbr=opts.use_bbr,
bold2t1w_dof=opts.bold2t1w_dof,
fmap_bspline=opts.fmap_bspline,
fmap_demean=opts.fmap_no_demean,
use_syn=opts.use_syn_sdc,
force_syn=opts.force_syn,
use_aroma=opts.use_aroma,
aroma_melodic_dim=opts.aroma_melodic_dimensionality,
ignore_aroma_err=opts.ignore_aroma_denoising_errors,
)
retval['return_code'] = 0
logs_path = Path(output_dir) / 'fmriprep' / 'logs'
boilerplate = retval['workflow'].visit_desc()
if boilerplate:
(logs_path / 'CITATION.md').write_text(boilerplate)
logger.log(
25, 'Works derived from this fMRIPrep execution should '
'include the following boilerplate:\n\n%s', boilerplate)
# Generate HTML file resolving citations
cmd = [
'pandoc', '-s', '--bibliography',
pkgrf('fmriprep',
'data/boilerplate.bib'), '--filter', 'pandoc-citeproc',
'--metadata', 'pagetitle="fMRIPrep citation boilerplate"',
str(logs_path / 'CITATION.md'), '-o',
str(logs_path / 'CITATION.html')
]
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning('Could not generate CITATION.html file:\n%s',
' '.join(cmd))
# Generate LaTex file resolving citations
cmd = [
'pandoc', '-s', '--bibliography',
pkgrf('fmriprep', 'data/boilerplate.bib'), '--natbib',
str(logs_path / 'CITATION.md'), '-o',
str(logs_path / 'CITATION.tex')
]
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning('Could not generate CITATION.tex file:\n%s',
' '.join(cmd))
else:
copyfile(pkgrf('fmriprep', 'data/boilerplate.bib'),
(logs_path / 'CITATION.bib'))
return retval
def build_workflow(opts, retval):
"""
Create the Nipype Workflow that supports the whole execution
graph, given the inputs.
All the checks and the construction of the workflow are done
inside this function that has pickleable inputs and output
dictionary (``retval``) to allow isolation using a
``multiprocessing.Process`` that allows fmriprep to enforce
a hard-limited memory-scope.
"""
from bids import BIDSLayout
from nipype import logging as nlogging, config as ncfg
from niworkflows.utils.bids import collect_participants
from niworkflows.reports import generate_reports
from ..__about__ import __version__
from ..workflows.base import init_fmriprep_wf
build_log = nlogging.getLogger('nipype.workflow')
INIT_MSG = """
Running fMRIPREP version {version}:
* BIDS dataset path: {bids_dir}.
* Participant list: {subject_list}.
* Run identifier: {uuid}.
""".format
bids_dir = opts.bids_dir.resolve()
output_dir = opts.output_dir.resolve()
work_dir = opts.work_dir.resolve()
retval['return_code'] = 1
retval['workflow'] = None
retval['bids_dir'] = str(bids_dir)
retval['output_dir'] = str(output_dir)
retval['work_dir'] = str(work_dir)
if output_dir == bids_dir:
build_log.error(
'The selected output folder is the same as the input BIDS folder. '
'Please modify the output path (suggestion: %s).', bids_dir /
'derivatives' / ('fmriprep-%s' % __version__.split('+')[0]))
retval['return_code'] = 1
return retval
output_spaces = parse_spaces(opts)
# Set up some instrumental utilities
run_uuid = '%s_%s' % (strftime('%Y%m%d-%H%M%S'), uuid.uuid4())
retval['run_uuid'] = run_uuid
# First check that bids_dir looks like a BIDS folder
layout = BIDSLayout(str(bids_dir),
validate=False,
ignore=("code", "stimuli", "sourcedata", "models",
"derivatives", re.compile(r'^\.')))
subject_list = collect_participants(
layout, participant_label=opts.participant_label)
retval['subject_list'] = subject_list
# Load base plugin_settings from file if --use-plugin
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as f:
plugin_settings = loadyml(f)
plugin_settings.setdefault('plugin_args', {})
else:
# Defaults
plugin_settings = {
'plugin': 'MultiProc',
'plugin_args': {
'raise_insufficient': False,
'maxtasksperchild': 1,
}
}
# Resource management options
# Note that we're making strong assumptions about valid plugin args
# This may need to be revisited if people try to use batch plugins
nthreads = plugin_settings['plugin_args'].get('n_procs')
# Permit overriding plugin config with specific CLI options
if nthreads is None or opts.nthreads is not None:
nthreads = opts.nthreads
if nthreads is None or nthreads < 1:
nthreads = cpu_count()
plugin_settings['plugin_args']['n_procs'] = nthreads
if opts.mem_mb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_mb / 1024
omp_nthreads = opts.omp_nthreads
if omp_nthreads == 0:
omp_nthreads = min(nthreads - 1 if nthreads > 1 else cpu_count(), 8)
if 1 < nthreads < omp_nthreads:
build_log.warning(
'Per-process threads (--omp-nthreads=%d) exceed total '
'threads (--nthreads/--n_cpus=%d)', omp_nthreads, nthreads)
retval['plugin_settings'] = plugin_settings
# Set up directories
log_dir = output_dir / 'fmriprep' / 'logs'
# Check and create output and working directories
output_dir.mkdir(exist_ok=True, parents=True)
log_dir.mkdir(exist_ok=True, parents=True)
work_dir.mkdir(exist_ok=True, parents=True)
# Nipype config (logs and execution)
ncfg.update_config({
'logging': {
'log_directory': str(log_dir),
'log_to_file': True
},
'execution': {
'crashdump_dir': str(log_dir),
'crashfile_format': 'txt',
'get_linked_libs': False,
'stop_on_first_crash': opts.stop_on_first_crash,
},
'monitoring': {
'enabled': opts.resource_monitor,
'sample_frequency': '0.5',
'summary_append': True,
}
})
if opts.resource_monitor:
ncfg.enable_resource_monitor()
# Called with reports only
if opts.reports_only:
build_log.log(25, 'Running --reports-only on participants %s',
', '.join(subject_list))
if opts.run_uuid is not None:
run_uuid = opts.run_uuid
retval['run_uuid'] = run_uuid
retval['return_code'] = generate_reports(subject_list,
output_dir,
work_dir,
run_uuid,
packagename='fmriprep')
return retval
# Build main workflow
build_log.log(
25,
INIT_MSG(version=__version__,
bids_dir=bids_dir,
subject_list=subject_list,
uuid=run_uuid))
retval['workflow'] = init_fmriprep_wf(
anat_only=opts.anat_only,
aroma_melodic_dim=opts.aroma_melodic_dimensionality,
bold2t1w_dof=opts.bold2t1w_dof,
cifti_output=opts.cifti_output,
debug=opts.sloppy,
dummy_scans=opts.dummy_scans,
echo_idx=opts.echo_idx,
err_on_aroma_warn=opts.error_on_aroma_warnings,
fmap_bspline=opts.fmap_bspline,
fmap_demean=opts.fmap_no_demean,
force_syn=opts.force_syn,
freesurfer=opts.run_reconall,
hires=opts.hires,
ignore=opts.ignore,
layout=layout,
longitudinal=opts.longitudinal,
low_mem=opts.low_mem,
medial_surface_nan=opts.medial_surface_nan,
omp_nthreads=omp_nthreads,
output_dir=str(output_dir),
output_spaces=output_spaces,
run_uuid=run_uuid,
regressors_all_comps=opts.return_all_components,
regressors_fd_th=opts.fd_spike_threshold,
regressors_dvars_th=opts.dvars_spike_threshold,
skull_strip_fixed_seed=opts.skull_strip_fixed_seed,
skull_strip_template=opts.skull_strip_template,
subject_list=subject_list,
t2s_coreg=opts.t2s_coreg,
task_id=opts.task_id,
use_aroma=opts.use_aroma,
use_bbr=opts.use_bbr,
use_syn=opts.use_syn_sdc,
work_dir=str(work_dir),
)
retval['return_code'] = 0
logs_path = Path(output_dir) / 'fmriprep' / 'logs'
boilerplate = retval['workflow'].visit_desc()
if boilerplate:
citation_files = {
ext: logs_path / ('CITATION.%s' % ext)
for ext in ('bib', 'tex', 'md', 'html')
}
# To please git-annex users and also to guarantee consistency
# among different renderings of the same file, first remove any
# existing one
for citation_file in citation_files.values():
try:
citation_file.unlink()
except FileNotFoundError:
pass
citation_files['md'].write_text(boilerplate)
build_log.log(
25, 'Works derived from this fMRIPrep execution should '
'include the following boilerplate:\n\n%s', boilerplate)
return retval
def test_masking(input_fname, expected_fname):
"""Check for regressions in masking."""
from nipype import config as ncfg
basename = Path(input_fname[0]).name
dsname = Path(expected_fname).parent.name
# Reconstruct base_fname from above
reports_dir = Path(os.getenv("FMRIPREP_REGRESSION_REPORTS", ""))
newpath = reports_dir / dsname
newpath.mkdir(parents=True, exist_ok=True)
# Nipype config (logs and execution)
ncfg.update_config({"execution": {
"crashdump_dir": str(newpath),
}})
wf = pe.Workflow(
name=basename.replace("_bold.nii.gz", "").replace("-", "_"))
base_dir = os.getenv("CACHED_WORK_DIRECTORY")
if base_dir:
base_dir = Path(base_dir) / dsname
base_dir.mkdir(parents=True, exist_ok=True)
wf.base_dir = str(base_dir)
epi_reference_wf = init_epi_reference_wf(omp_nthreads=os.cpu_count(),
auto_bold_nss=True)
epi_reference_wf.inputs.inputnode.in_files = input_fname
enhance_and_skullstrip_bold_wf = init_enhance_and_skullstrip_bold_wf()
out_fname = fname_presuffix(Path(expected_fname).name,
suffix=".svg",
use_ext=False,
newpath=str(newpath))
mask_diff_plot = pe.Node(ROIsPlot(colors=["limegreen"], levels=[0.5]),
name="mask_diff_plot")
mask_diff_plot.always_run = True
mask_diff_plot.inputs.in_mask = expected_fname
mask_diff_plot.inputs.out_report = out_fname
# fmt:off
wf.connect([
(epi_reference_wf, enhance_and_skullstrip_bold_wf,
[("outputnode.epi_ref_file", "inputnode.in_file")]),
(enhance_and_skullstrip_bold_wf, mask_diff_plot, [
("outputnode.bias_corrected_file", "in_file"),
("outputnode.mask_file", "in_rois"),
]),
])
res = wf.run(plugin="MultiProc")
combine_masks = [
node for node in res.nodes if node.name.endswith("combine_masks")
][0]
overlap = symmetric_overlap(expected_fname,
combine_masks.result.outputs.out_file)
mask_dir = reports_dir / "fmriprep_bold_mask" / dsname
mask_dir.mkdir(parents=True, exist_ok=True)
copyfile(
combine_masks.result.outputs.out_file,
str(mask_dir / Path(expected_fname).name),
copy=True,
)
assert overlap > 0.95, input_fname
######################
# WF
######################
wd_dir = '/scr/kansas1/data/lsd-lemon/lemon_wd_meanDist_%s' % distype
ds_dir = '/scr/kansas1/data/lsd-lemon/lemon_results_meanDist_%s' % distype
wf = Workflow(name='distconnect_meanDist_%s' % distype)
wf.base_dir = os.path.join(wd_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': False,
'remove_unnecessary_outputs': False,
'job_finished_timeout': 120})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(wd_dir, 'crash')
ds = Node(nio.DataSink(base_directory=ds_dir), name='ds')
######################
# GET DATA
######################
# SUBJECTS ITERATOR
subjects_infosource = Node(util.IdentityInterface(fields=['subject_id']), name='subjects_infosource')
subjects_infosource.iterables = ('subject_id', subjects_list)
run_mean_dist = Node(util.Function(input_names=['sub'],
output_names=[],
def calc_local_metrics(
preprocessed_data_dir,
subject_id,
parcellations_dict,
bp_freq_list,
fd_thresh,
working_dir,
ds_dir,
use_n_procs,
plugin_name,
):
import os
from nipype import config
from nipype.pipeline.engine import Node, Workflow, MapNode
import nipype.interfaces.utility as util
import nipype.interfaces.io as nio
import nipype.interfaces.fsl as fsl
import utils as calc_metrics_utils
#####################################
# GENERAL SETTINGS
#####################################
fsl.FSLCommand.set_default_output_type("NIFTI_GZ")
wf = Workflow(name="LeiCA_LIFE_metrics")
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(
logging=dict(workflow_level="DEBUG"),
execution={"stop_on_first_crash": True, "remove_unnecessary_outputs": True, "job_finished_timeout": 15},
)
config.update_config(nipype_cfg)
wf.config["execution"]["crashdump_dir"] = os.path.join(working_dir, "crash")
ds = Node(nio.DataSink(base_directory=ds_dir), name="ds")
ds.inputs.regexp_substitutions = [
("MNI_resampled_brain_mask_calc.nii.gz", "falff.nii.gz"),
("residual_filtered_3dT.nii.gz", "alff.nii.gz"),
("_parcellation_", ""),
("_bp_freqs_", "bp_"),
]
#####################
# ITERATORS
#####################
# PARCELLATION ITERATOR
parcellation_infosource = Node(util.IdentityInterface(fields=["parcellation"]), name="parcellation_infosource")
parcellation_infosource.iterables = ("parcellation", parcellations_dict.keys())
bp_filter_infosource = Node(util.IdentityInterface(fields=["bp_freqs"]), name="bp_filter_infosource")
bp_filter_infosource.iterables = ("bp_freqs", bp_freq_list)
selectfiles = Node(
nio.SelectFiles(
{
"parcellation_time_series": "{subject_id}/con_mat/parcellated_time_series/bp_{bp_freqs}/{parcellation}/parcellation_time_series.npy"
},
base_directory=preprocessed_data_dir,
),
name="selectfiles",
)
selectfiles.inputs.subject_id = subject_id
wf.connect(parcellation_infosource, "parcellation", selectfiles, "parcellation")
wf.connect(bp_filter_infosource, "bp_freqs", selectfiles, "bp_freqs")
fd_file = Node(
nio.SelectFiles({"fd_p": "{subject_id}/QC/FD_P_ts"}, base_directory=preprocessed_data_dir), name="fd_file"
)
fd_file.inputs.subject_id = subject_id
##############
## CON MATS
##############
##############
## extract ts
##############
get_good_trs = Node(
util.Function(
input_names=["fd_file", "fd_thresh"],
output_names=["good_trs", "fd_scrubbed_file"],
function=calc_metrics_utils.get_good_trs,
),
name="get_good_trs",
)
wf.connect(fd_file, "fd_p", get_good_trs, "fd_file")
get_good_trs.inputs.fd_thresh = fd_thresh
parcellated_ts_scrubbed = Node(
util.Function(
input_names=["parcellation_time_series_file", "good_trs"],
output_names=["parcellation_time_series_scrubbed"],
function=calc_metrics_utils.parcellation_time_series_scrubbing,
),
name="parcellated_ts_scrubbed",
)
wf.connect(selectfiles, "parcellation_time_series", parcellated_ts_scrubbed, "parcellation_time_series_file")
wf.connect(get_good_trs, "good_trs", parcellated_ts_scrubbed, "good_trs")
##############
## get conmat
##############
con_mat = Node(
util.Function(
input_names=["in_data", "extraction_method"],
output_names=["matrix", "matrix_file"],
function=calc_metrics_utils.calculate_connectivity_matrix,
),
name="con_mat",
)
con_mat.inputs.extraction_method = "correlation"
wf.connect(parcellated_ts_scrubbed, "parcellation_time_series_scrubbed", con_mat, "in_data")
##############
## ds
##############
wf.connect(get_good_trs, "fd_scrubbed_file", ds, "QC.@fd_scrubbed_file")
fd_str = ("%.1f" % fd_thresh).replace(".", "_")
wf.connect(con_mat, "matrix_file", ds, "con_mat.matrix_scrubbed_%s.@mat" % fd_str)
# wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical')
# wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf')
# wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf')
if plugin_name == "CondorDAGMan":
wf.run(plugin=plugin_name, plugin_args={"initial_specs": "request_memory = 1500"})
if plugin_name == "MultiProc":
wf.run(plugin=plugin_name, plugin_args={"n_procs": use_n_procs})
from nipype import config
import clinica.pipelines.engine as cpe
# Use hash instead of parameters for iterables folder names
# Otherwise path will be too long and generate OSError
cfg = dict(execution={"parameterize_dirs": False})
config.update_config(cfg)
class PETVolume(cpe.Pipeline):
"""PETVolume - Volume-based processing of PET images using SPM.
Returns:
A clinica pipeline object containing the PETVolume pipeline.
"""
def check_pipeline_parameters(self):
"""Check pipeline parameters."""
from clinica.utils.atlas import PET_VOLUME_ATLASES
from clinica.utils.group import check_group_label
self.parameters.setdefault("group_label", None)
check_group_label(self.parameters["group_label"])
if "acq_label" not in self.parameters.keys():
raise KeyError(
"Missing compulsory acq_label key in pipeline parameter.")
self.parameters.setdefault("pvc_psf_tsv", None)
self.parameters.setdefault("mask_tissues", [1, 2, 3])
self.parameters.setdefault("mask_threshold", 0.3)
self.parameters.setdefault("pvc_mask_tissues", [1, 2, 3])
def preprocessing_pipeline(cfg):
import os
from nipype import config
from nipype.pipeline.engine import Node, Workflow
import nipype.interfaces.utility as util
import nipype.interfaces.io as nio
import nipype.interfaces.fsl as fsl
import nipype.interfaces.freesurfer as freesurfer
# LeiCA modules
from utils import zip_and_save_running_scripts
from preprocessing.rsfMRI_preprocessing import create_rsfMRI_preproc_pipeline
from preprocessing.converter import create_converter_structural_pipeline, create_converter_functional_pipeline, \
create_converter_diffusion_pipeline
# INPUT PARAMETERS
dicom_dir = cfg['dicom_dir']
working_dir = cfg['working_dir']
freesurfer_dir = cfg['freesurfer_dir']
template_dir = cfg['template_dir']
script_dir = cfg['script_dir']
ds_dir = cfg['ds_dir']
subject_id = cfg['subject_id']
TR_list = cfg['TR_list']
vols_to_drop = cfg['vols_to_drop']
lp_cutoff_freq = cfg['lp_cutoff_freq']
hp_cutoff_freq = cfg['hp_cutoff_freq']
use_fs_brainmask = cfg['use_fs_brainmask']
use_n_procs = cfg['use_n_procs']
plugin_name = cfg['plugin_name']
#####################################
# GENERAL SETTINGS
#####################################
fsl.FSLCommand.set_default_output_type('NIFTI_GZ')
freesurfer.FSCommand.set_default_subjects_dir(freesurfer_dir)
wf = Workflow(name='LeiCA_resting')
wf.base_dir = os.path.join(working_dir)
nipype_cfg = dict(logging=dict(workflow_level='DEBUG'),
execution={
'stop_on_first_crash': True,
'remove_unnecessary_outputs': True,
'job_finished_timeout': 120
})
config.update_config(nipype_cfg)
wf.config['execution']['crashdump_dir'] = os.path.join(
working_dir, 'crash')
ds = Node(nio.DataSink(base_directory=ds_dir), name='ds')
#####################################
# SET ITERATORS
#####################################
# GET SCAN TR_ID ITERATOR
scan_infosource = Node(util.IdentityInterface(fields=['TR_id']),
name='scan_infosource')
scan_infosource.iterables = ('TR_id', TR_list)
#####################################
# FETCH MRI DATA
#####################################
# GET LATERAL VENTRICLE MASK
templates_atlases = {
'lat_ventricle_mask_MNI':
'cpac_image_resources/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz'
}
selectfiles_templates = Node(nio.SelectFiles(templates_atlases,
base_directory=template_dir),
name="selectfiles_templates")
if not True: # releases 1-6 with 01... format subject_id
# GET FUNCTIONAL DATA
templates_funct = {
'funct_dicom': '{subject_id}/session_1/RfMRI_*_{TR_id}'
}
selectfiles_funct = Node(nio.SelectFiles(templates_funct,
base_directory=dicom_dir),
name="selectfiles_funct")
selectfiles_funct.inputs.subject_id = subject_id
wf.connect(scan_infosource, 'TR_id', selectfiles_funct, 'TR_id')
# GET STRUCTURAL DATA
templates_struct = {
't1w_dicom': '{subject_id}/anat',
'dMRI_dicom': '{subject_id}/session_1/DTI_mx_137/*.dcm'
} # *.dcm for dMRI as Dcm2nii requires this
selectfiles_struct = Node(nio.SelectFiles(templates_struct,
base_directory=dicom_dir),
name="selectfiles_struct")
selectfiles_struct.inputs.subject_id = subject_id
else: #startin with release 6 new folder structure
templates_funct = {
'funct_dicom': '*/{subject_id}/*_V2/REST_{TR_id}*/*.dcm'
}
selectfiles_funct = Node(nio.SelectFiles(templates_funct,
base_directory=dicom_dir),
name="selectfiles_funct")
selectfiles_funct.inputs.subject_id = subject_id
wf.connect(scan_infosource, 'TR_id', selectfiles_funct, 'TR_id')
# GET STRUCTURAL DATA
templates_struct = {
't1w_dicom': '*/{subject_id}/*_V2/MPRAGE_SIEMENS_DEFACED*/*.dcm',
'dMRI_dicom': '*/{subject_id}/*_V2/DIFF_137_AP*/*.dcm'
} # *.dcm for dMRI as Dcm2nii requires this
selectfiles_struct = Node(nio.SelectFiles(templates_struct,
base_directory=dicom_dir),
name="selectfiles_struct")
selectfiles_struct.inputs.subject_id = subject_id
#####################################
# COPY RUNNING SCRIPTS
#####################################
copy_scripts = Node(util.Function(input_names=['subject_id', 'script_dir'],
output_names=['zip_file'],
function=zip_and_save_running_scripts),
name='copy_scripts')
copy_scripts.inputs.script_dir = script_dir
copy_scripts.inputs.subject_id = subject_id
wf.connect(copy_scripts, 'zip_file', ds, 'scripts')
#####################################
# CONVERT DICOMs
#####################################
# CONVERT STRUCT 2 NIFTI
converter_struct = create_converter_structural_pipeline(
working_dir, ds_dir, 'converter_struct')
wf.connect(selectfiles_struct, 't1w_dicom', converter_struct,
'inputnode.t1w_dicom')
# CONVERT dMRI 2 NIFTI
converter_dMRI = create_converter_diffusion_pipeline(
working_dir, ds_dir, 'converter_dMRI')
wf.connect(selectfiles_struct, 'dMRI_dicom', converter_dMRI,
'inputnode.dMRI_dicom')
# CONVERT FUNCT 2 NIFTI
converter_funct = create_converter_functional_pipeline(
working_dir, ds_dir, 'converter_funct')
wf.connect(selectfiles_funct, 'funct_dicom', converter_funct,
'inputnode.epi_dicom')
wf.connect(scan_infosource, 'TR_id', converter_funct,
'inputnode.out_format')
#####################################
# START RSFMRI PREPROCESSING ANALYSIS
#####################################
# rsfMRI PREPROCESSING
rsfMRI_preproc = create_rsfMRI_preproc_pipeline(working_dir,
freesurfer_dir, ds_dir,
use_fs_brainmask,
'rsfMRI_preprocessing')
rsfMRI_preproc.inputs.inputnode.vols_to_drop = vols_to_drop
rsfMRI_preproc.inputs.inputnode.lp_cutoff_freq = lp_cutoff_freq
rsfMRI_preproc.inputs.inputnode.hp_cutoff_freq = hp_cutoff_freq
rsfMRI_preproc.inputs.inputnode.subject_id = subject_id
wf.connect(converter_struct, 'outputnode.t1w', rsfMRI_preproc,
'inputnode.t1w')
wf.connect(converter_funct, 'outputnode.epi', rsfMRI_preproc,
'inputnode.epi')
wf.connect(converter_funct, 'outputnode.TR_ms', rsfMRI_preproc,
'inputnode.TR_ms')
wf.connect(selectfiles_templates, 'lat_ventricle_mask_MNI', rsfMRI_preproc,
'inputnode.lat_ventricle_mask_MNI')
#####################################
# RUN WF
#####################################
wf.write_graph(dotfilename=wf.name, graph2use='colored',
format='pdf') # 'hierarchical')
wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf')
wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf')
if plugin_name == 'CondorDAGMan':
wf.run(plugin=plugin_name)
if plugin_name == 'MultiProc':
wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})