def get_subjects_sessions_dictionary(input_subjects,
cache,
resultdir,
prefix,
dbfile,
useSentinal,
shuffle=False):
import random
_temp = OpenSubjectDatabase(cache, ['all'], prefix, dbfile)
if "all" in input_subjects:
input_subjects = _temp.getAllSubjects()
if useSentinal:
print("=" * 80)
print("Using Sentinal Files to Limit Jobs Run")
_all_subjects = set(input_subjects)
_processed_subjects = set(
get_processed_subjects(resultdir, input_subjects))
subjects = list(_all_subjects - _processed_subjects
) #NOTE - in set operation notation removes values
else:
subjects = input_subjects
if shuffle:
random.shuffle(
subjects) # randomly shuffle to get max cluster efficiency
subject_sessions_dictionary = dict()
for subject in subjects:
subject_sessions_dictionary[subject] = _temp.getSessionsFromSubject(
subject)
return subjects, subject_sessions_dictionary
def get_subjects_sessions_dictionary(input_subjects, cache, resultdir, prefix, dbfile, useSentinal, shuffle=False):
import random
_temp = OpenSubjectDatabase(cache, ['all'], prefix, dbfile)
if "all" in input_subjects:
input_subjects = _temp.getAllSubjects();
if useSentinal:
print("="*80)
print("Using Sentinal Files to Limit Jobs Run")
_all_subjects = set(input_subjects )
_processed_subjects = set( get_processed_subjects( resultdir, input_subjects ) )
subjects = list( _all_subjects - _processed_subjects ) #NOTE - in set operation notation removes values
else:
subjects = input_subjects
if shuffle:
random.shuffle(subjects) # randomly shuffle to get max cluster efficiency
subject_sessions_dictionary = dict()
for subject in subjects:
subject_sessions_dictionary[subject]=_temp.getSessionsFromSubject(subject)
return subjects,subject_sessions_dictionary
def RunSubjectWorkflow(args):
"""
.-----------.
--- | Session 1 | ---> /project/subjectA/session1/phase/
/ *-----------*
.-----------. /
| Subject A | <
*-----------* \
\ .-----------.
--- | Session 2 | ---> /project/subjectA/session2/phase/
*-----------*
**** Replaces WorkflowT1T2.py ****
"""
start_time, subject, master_config = args
assert 'baseline' in master_config[
'components'] or 'longitudinal' in master_config[
'components'], "Baseline or Longitudinal is not in WORKFLOW_COMPONENTS!"
import time
from nipype import config, logging
config.update_config(master_config) # Set universal pipeline options
assert config.get('execution',
'plugin') == master_config['execution']['plugin']
# DEBUG
# config.enable_debug_mode()
# config.set('execution', 'stop_on_first_rerun', 'true')
# END DEBUG
logging.update_logging(config)
import nipype.pipeline.engine as pe
import nipype.interfaces.base as nbase
import nipype.interfaces.io as nio
from nipype.interfaces.utility import IdentityInterface, Function
import traits
from baw_exp import OpenSubjectDatabase
from SessionDB import SessionDB
from PipeLineFunctionHelpers import convertToList
from atlasNode import MakeAtlasNode
from utilities.misc import GenerateSubjectOutputPattern as outputPattern
from utilities.misc import GenerateWFName
while time.time() < start_time:
time.sleep(start_time - time.time() + 1)
print "Delaying start for {subject}".format(subject=subject)
print("===================== SUBJECT: {0} ===========================".
format(subject))
subjectWorkflow = pe.Workflow(
name="BAW_StandardWorkup_subject_{0}".format(subject))
subjectWorkflow.base_dir = config.get('logging', 'log_directory')
# subjectWorkflow.config['execution']['plugin'] = 'Linear' # Hardcodeded in WorkupT1T2.py - why?
# DEBUG
# subjectWorkflow.config['execution']['stop_on_first_rerun'] = 'true'
# END DEBUG
atlasNode = MakeAtlasNode(master_config['atlascache'], 'BAtlas')
sessionWorkflow = dict()
inputsSpec = dict()
# To avoid a "sqlite3.ProgrammingError: Base Cursor.__init__ not called" error
# using multiprocessing.map_async(), instantiate database here
database = OpenSubjectDatabase(master_config['cachedir'], [subject],
master_config['prefix'],
master_config['dbfile'])
# print database.getAllSessions()
database.open_connection()
sessions = database.getSessionsFromSubject(subject)
# print "These are the sessions: ", sessions
if 'baseline' in master_config['components']:
current_phase = 'baseline'
from baseline import baseline_workflow as create_wkfl
elif 'longitudinal' in master_config['components']:
current_phase = 'longitudinal'
from longitudinal import create_longitudinal as create_wkfl
for session in sessions: # TODO (future): Replace with iterable inputSpec node and add Function node for getAllFiles()
project = database.getProjFromSession(session)
pname = "{0}_{1}".format(
session,
current_phase) # Long node names make graphs a pain to read/print
# pname = GenerateWFName(project, subject, session, current_phase)
print "Building session pipeline for {0}".format(session)
inputsSpec[session] = pe.Node(
name='inputspec_{0}'.format(session),
interface=IdentityInterface(
fields=['T1s', 'T2s', 'PDs', 'FLs', 'OTs']))
inputsSpec[session].inputs.T1s = database.getFilenamesByScantype(
session, ['T1-15', 'T1-30'])
inputsSpec[session].inputs.T2s = database.getFilenamesByScantype(
session, ['T2-15', 'T2-30'])
inputsSpec[session].inputs.PDs = database.getFilenamesByScantype(
session, ['PD-15', 'PD-30'])
inputsSpec[session].inputs.FLs = database.getFilenamesByScantype(
session, ['FL-15', 'FL-30'])
inputsSpec[session].inputs.OTs = database.getFilenamesByScantype(
session, ['OTHER-15', 'OTHER-30'])
sessionWorkflow[session] = create_wkfl(project,
subject,
session,
master_config,
interpMode='Linear',
pipeline_name=pname)
subjectWorkflow.connect([
(inputsSpec[session], sessionWorkflow[session], [
('T1s', 'inputspec.T1s'),
('T2s', 'inputspec.T2s'),
('PDs', 'inputspec.PDs'),
('FLs', 'inputspec.FLs'),
('OTs', 'inputspec.OTHERs'),
]),
(atlasNode, sessionWorkflow[session],
[('template_landmarks_50Lmks_fcsv',
'inputspec.atlasLandmarkFilename'),
('template_weights_50Lmks_wts', 'inputspec.atlasWeightFilename'),
('LLSModel_50Lmks_hdf5', 'inputspec.LLSModel'),
('T1_50Lmks_mdl', 'inputspec.inputTemplateModel')]),
])
if current_phase == 'baseline':
subjectWorkflow.connect([
(atlasNode, sessionWorkflow[session],
[('template_t1', 'inputspec.template_t1'),
('ExtendedAtlasDefinition_xml', 'inputspec.atlasDefinition')
]),
])
else:
assert current_phase == 'longitudinal', "Phase value is unknown: {0}".format(
current_phase)
from utils import run_workflow, print_workflow
if False:
print_workflow(subjectWorkflow,
plugin=master_config['execution']['plugin'],
dotfilename='subjectWorkflow')
return run_workflow(subjectWorkflow,
plugin=master_config['execution']['plugin'],
plugin_args=master_config['plugin_args'])
def RunSubjectWorkflow(args):
"""
.-----------.
--- | Session 1 | ---> /project/subjectA/session1/phase/
/ *-----------*
.-----------. /
| Subject A | <
*-----------* \
\ .-----------.
--- | Session 2 | ---> /project/subjectA/session2/phase/
*-----------*
**** Replaces WorkflowT1T2.py ****
"""
start_time, subject, master_config = args
assert 'tissue_classify' in master_config['components'] or 'auxlmk' in master_config['components'] or 'segmentation' in master_config['components'], "Baseline or Longitudinal is not in WORKFLOW_COMPONENTS!"
import time
from nipype import config, logging
config.update_config(master_config) # Set universal pipeline options
assert config.get('execution', 'plugin') == master_config['execution']['plugin']
# DEBUG
# config.enable_debug_mode()
# config.set('execution', 'stop_on_first_rerun', 'true')
# END DEBUG
logging.update_logging(config)
import nipype.pipeline.engine as pe
import nipype.interfaces.base as nbase
import nipype.interfaces.io as nio
from nipype.interfaces.utility import IdentityInterface, Function
import traits
from baw_exp import OpenSubjectDatabase
from SessionDB import SessionDB
from PipeLineFunctionHelpers import convertToList
from atlasNode import MakeAtlasNode
from utilities.misc import GenerateSubjectOutputPattern as outputPattern
from utilities.misc import GenerateWFName
from utils import run_workflow, print_workflow
# while time.time() < start_time:
# time.sleep(start_time - time.time() + 1)
# print "Delaying start for {subject}".format(subject=subject)
# print("===================== SUBJECT: {0} ===========================".format(subject))
subjectWorkflow = pe.Workflow(name="BAW_StandardWorkup_subject_{0}".format(subject))
subjectWorkflow.base_dir = config.get('logging', 'log_directory')
# subjectWorkflow.config['execution']['plugin'] = 'Linear' # Hardcodeded in WorkupT1T2.py - why?
# DEBUG
# subjectWorkflow.config['execution']['stop_on_first_rerun'] = 'true'
# END DEBUG
sessionWorkflow = dict()
inputsSpec = dict()
# To avoid a "sqlite3.ProgrammingError: Base Cursor.__init__ not called" error
# using multiprocessing.map_async(), instantiate database here
database = OpenSubjectDatabase(master_config['cachedir'], [subject], master_config['prefix'], master_config['dbfile'])
# print database.getAllSessions()
database.open_connection()
sessions = database.getSessionsFromSubject(subject)
print "These are the sessions: ", sessions
# TODO: atlas input csv read
atlasNode = MakeAtlasNode(master_config['atlascache'], 'BAtlas')
# atlasNode = GetAtlasNode(master_config['previouscache'], 'BAtlas')
from singleSession import create_singleSession as create_wkfl
for session in sessions: # TODO (future): Replace with iterable inputSpec node and add Function node for getAllFiles()
project = database.getProjFromSession(session)
pname = "{0}_singleSession".format(session) # Long node names make graphs a pain to read/print
# pname = GenerateWFName(project, subject, session, 'singleSession')
print "Building session pipeline for {0}".format(session)
inputsSpec[session] = pe.Node(name='inputspec_{0}'.format(session),
interface=IdentityInterface(fields=['T1s', 'T2s', 'PDs', 'FLs', 'OTs']))
inputsSpec[session].inputs.T1s = database.getFilenamesByScantype(session, ['T1-15', 'T1-30'])
inputsSpec[session].inputs.T2s = database.getFilenamesByScantype(session, ['T2-15', 'T2-30'])
inputsSpec[session].inputs.PDs = database.getFilenamesByScantype(session, ['PD-15', 'PD-30'])
inputsSpec[session].inputs.FLs = database.getFilenamesByScantype(session, ['FL-15', 'FL-30'])
inputsSpec[session].inputs.OTs = database.getFilenamesByScantype(session, ['OTHER-15', 'OTHER-30'])
sessionWorkflow[session] = create_wkfl(project, subject, session, master_config,
interpMode='Linear', pipeline_name=pname)
subjectWorkflow.connect([(inputsSpec[session], sessionWorkflow[session], [('T1s', 'inputspec.T1s'),
('T2s', 'inputspec.T2s'),
('PDs', 'inputspec.PDs'),
('FLs', 'inputspec.FLs'),
('OTs', 'inputspec.OTHERs'),
]),
(atlasNode, sessionWorkflow[session], [('template_landmarks_50Lmks_fcsv',
'inputspec.atlasLandmarkFilename'),
('template_weights_50Lmks_wts',
'inputspec.atlasWeightFilename'),
('LLSModel_50Lmks_hdf5', 'inputspec.LLSModel'),
('T1_50Lmks_mdl', 'inputspec.inputTemplateModel')]),
])
if 'segmentation' in master_config['components']:
from WorkupT1T2BRAINSCut import GenerateWFName
try:
bCutInputName = ".".join(['segmentation', GenerateWFName(project, subject, session, 'Segmentation'), 'inputspec'])
except:
print project, subject, session
raise
subjectWorkflow.connect([(atlasNode, sessionWorkflow[session],
[('hncma-atlas', 'segmentation.inputspec.hncma-atlas'),
('template_t1', 'segmentation.inputspec.template_t1'),
('template_t1', bCutInputName + '.template_t1'),
('rho', bCutInputName + '.rho'),
('phi', bCutInputName + '.phi'),
('theta', bCutInputName + '.theta'),
('l_caudate_ProbabilityMap', bCutInputName + '.l_caudate_ProbabilityMap'),
('r_caudate_ProbabilityMap', bCutInputName + '.r_caudate_ProbabilityMap'),
('l_hippocampus_ProbabilityMap', bCutInputName + '.l_hippocampus_ProbabilityMap'),
('r_hippocampus_ProbabilityMap', bCutInputName + '.r_hippocampus_ProbabilityMap'),
('l_putamen_ProbabilityMap', bCutInputName + '.l_putamen_ProbabilityMap'),
('r_putamen_ProbabilityMap', bCutInputName + '.r_putamen_ProbabilityMap'),
('l_thalamus_ProbabilityMap', bCutInputName + '.l_thalamus_ProbabilityMap'),
('r_thalamus_ProbabilityMap', bCutInputName + '.r_thalamus_ProbabilityMap'),
('l_accumben_ProbabilityMap', bCutInputName + '.l_accumben_ProbabilityMap'),
('r_accumben_ProbabilityMap', bCutInputName + '.r_accumben_ProbabilityMap'),
('l_globus_ProbabilityMap', bCutInputName + '.l_globus_ProbabilityMap'),
('r_globus_ProbabilityMap', bCutInputName + '.r_globus_ProbabilityMap'),
('trainModelFile_txtD0060NT0060_gz',
bCutInputName + '.trainModelFile_txtD0060NT0060_gz')])])
if True: # FIXME: current_phase == 'baseline':
subjectWorkflow.connect([(atlasNode, sessionWorkflow[session], [('template_t1', 'inputspec.template_t1'),
('ExtendedAtlasDefinition_xml',
'inputspec.atlasDefinition')]),
])
else:
template_DG = pe.Node(interface=nio.DataGrabber(infields=['subject'],
outfields=['template_t1', 'outAtlasFullPath']),
name='Template_DG')
template_DG.inputs.base_directory = master_config['previousresult']
template_DG.inputs.subject = subject
template_DG.inputs.template = 'SUBJECT_TEMPLATES/%s/AVG_%s.nii.gz'
template_DG.inputs.template_args['template_t1'] = [['subject', 'T1']]
template_DG.inputs.field_template = {'outAtlasFullPath': 'Atlas/definitions/AtlasDefinition_%s.xml'}
template_DG.inputs.template_args['outAtlasFullPath'] = [['subject']]
template_DG.inputs.sort_filelist = True
template_DG.inputs.raise_on_empty = True
baw201.connect([(template_DG, sessionWorkflow[session], [('outAtlasFullPath', 'inputspec.atlasDefinition'),
('template_t1', 'inputspec.template_t1')]),
])
# HACK: only run first subject
break
# END HACK
if not True:
return print_workflow(subjectWorkflow,
plugin=master_config['execution']['plugin'], dotfilename='subjectWorkflow') #, graph2use='flat')
try:
return subjectWorkflow.run(plugin='SGEGraph', plugin_args=master_config['plugin_args'])
except:
return 1
def RunSubjectWorkflow(args):
"""
.-----------.
--- | Session 1 | ---> /project/subjectA/session1/phase/
/ *-----------*
.-----------. /
| Subject A | <
*-----------* \
\ .-----------.
--- | Session 2 | ---> /project/subjectA/session2/phase/
*-----------*
**** Replaces WorkflowT1T2.py ****
"""
start_time, subject, master_config = args
assert 'baseline' in master_config['components'] or 'longitudinal' in master_config['components'], "Baseline or Longitudinal is not in WORKFLOW_COMPONENTS!"
import time
from nipype import config, logging
config.update_config(master_config) # Set universal pipeline options
assert config.get('execution', 'plugin') == master_config['execution']['plugin']
# DEBUG
# config.enable_debug_mode()
# config.set('execution', 'stop_on_first_rerun', 'true')
# END DEBUG
logging.update_logging(config)
import nipype.pipeline.engine as pe
import nipype.interfaces.base as nbase
import nipype.interfaces.io as nio
from nipype.interfaces.utility import IdentityInterface, Function
import traits
from baw_exp import OpenSubjectDatabase
from SessionDB import SessionDB
from PipeLineFunctionHelpers import convertToList
from atlasNode import MakeAtlasNode
from utilities.misc import GenerateSubjectOutputPattern as outputPattern
from utilities.misc import GenerateWFName
while time.time() < start_time:
time.sleep(start_time - time.time() + 1)
print "Delaying start for {subject}".format(subject=subject)
print("===================== SUBJECT: {0} ===========================".format(subject))
subjectWorkflow = pe.Workflow(name="BAW_StandardWorkup_subject_{0}".format(subject))
subjectWorkflow.base_dir = config.get('logging', 'log_directory')
# subjectWorkflow.config['execution']['plugin'] = 'Linear' # Hardcodeded in WorkupT1T2.py - why?
# DEBUG
# subjectWorkflow.config['execution']['stop_on_first_rerun'] = 'true'
# END DEBUG
atlasNode = MakeAtlasNode(master_config['atlascache'], 'BAtlas')
sessionWorkflow = dict()
inputsSpec = dict()
# To avoid a "sqlite3.ProgrammingError: Base Cursor.__init__ not called" error
# using multiprocessing.map_async(), instantiate database here
database = OpenSubjectDatabase(master_config['cachedir'], [subject], master_config['prefix'], master_config['dbfile'])
# print database.getAllSessions()
database.open_connection()
sessions = database.getSessionsFromSubject(subject)
# print "These are the sessions: ", sessions
if 'baseline' in master_config['components']:
current_phase = 'baseline'
from baseline import baseline_workflow as create_wkfl
elif 'longitudinal' in master_config['components']:
current_phase = 'longitudinal'
from longitudinal import create_longitudinal as create_wkfl
for session in sessions: # TODO (future): Replace with iterable inputSpec node and add Function node for getAllFiles()
project = database.getProjFromSession(session)
pname = "{0}_{1}".format(session, current_phase) # Long node names make graphs a pain to read/print
# pname = GenerateWFName(project, subject, session, current_phase)
print "Building session pipeline for {0}".format(session)
inputsSpec[session] = pe.Node(name='inputspec_{0}'.format(session),
interface=IdentityInterface(fields=['T1s', 'T2s', 'PDs', 'FLs', 'OTs']))
inputsSpec[session].inputs.T1s = database.getFilenamesByScantype(session, ['T1-15', 'T1-30'])
inputsSpec[session].inputs.T2s = database.getFilenamesByScantype(session, ['T2-15', 'T2-30'])
inputsSpec[session].inputs.PDs = database.getFilenamesByScantype(session, ['PD-15', 'PD-30'])
inputsSpec[session].inputs.FLs = database.getFilenamesByScantype(session, ['FL-15', 'FL-30'])
inputsSpec[session].inputs.OTs = database.getFilenamesByScantype(session, ['OTHER-15', 'OTHER-30'])
sessionWorkflow[session] = create_wkfl(project, subject, session, master_config,
interpMode='Linear', pipeline_name=pname)
subjectWorkflow.connect([(inputsSpec[session], sessionWorkflow[session], [('T1s', 'inputspec.T1s'),
('T2s', 'inputspec.T2s'),
('PDs', 'inputspec.PDs'),
('FLs', 'inputspec.FLs'),
('OTs', 'inputspec.OTHERs'),
]),
(atlasNode, sessionWorkflow[session], [('template_landmarks_50Lmks_fcsv',
'inputspec.atlasLandmarkFilename'),
('template_weights_50Lmks_wts',
'inputspec.atlasWeightFilename'),
('LLSModel_50Lmks_hdf5', 'inputspec.LLSModel'),
('T1_50Lmks_mdl', 'inputspec.inputTemplateModel')]),
])
if current_phase == 'baseline':
subjectWorkflow.connect([(atlasNode, sessionWorkflow[session], [('template_t1', 'inputspec.template_t1'),
('ExtendedAtlasDefinition_xml',
'inputspec.atlasDefinition')]),
])
else:
assert current_phase == 'longitudinal', "Phase value is unknown: {0}".format(current_phase)
from utils import run_workflow, print_workflow
if False:
print_workflow(subjectWorkflow, plugin=master_config['execution']['plugin'], dotfilename='subjectWorkflow')
return run_workflow(subjectWorkflow, plugin=master_config['execution']['plugin'], plugin_args=master_config['plugin_args'])
