def run(config_file,
subject_list_file,
p_name=None,
plugin=None,
plugin_args=None,
tracking=True,
num_subs_at_once=None):
'''
'''
# Import packages
import commands
import os
import pickle
import time
from CPAC.pipeline.cpac_pipeline import prep_workflow
# Init variables
config_file = os.path.realpath(config_file)
subject_list_file = os.path.realpath(subject_list_file)
# take date+time stamp for run identification purposes
unique_pipeline_id = strftime("%Y%m%d%H%M%S")
pipeline_start_stamp = strftime("%Y-%m-%d_%H:%M:%S")
# Load in pipeline config file
try:
if not os.path.exists(config_file):
raise IOError
else:
c = Configuration(yaml.load(open(config_file, 'r')))
except IOError:
print "config file %s doesn't exist" % config_file
raise
except Exception as e:
raise Exception("Error reading config file - {0}\n\nError details:"
"\n{1}\n\n".format(config_file, e))
c.logDirectory = os.path.abspath(c.logDirectory)
c.workingDirectory = os.path.abspath(c.workingDirectory)
c.outputDirectory = os.path.abspath(c.outputDirectory)
c.crashLogDirectory = os.path.abspath(c.crashLogDirectory)
if num_subs_at_once:
if not str(num_subs_at_once).isdigit():
raise Exception('[!] Value entered for --num_cores not a digit.')
c.numParticipantsAtOnce = int(num_subs_at_once)
# Do some validation
validate(c)
# Get the pipeline name
p_name = p_name or c.pipelineName
# Load in subject list
try:
with open(subject_list_file, 'r') as sf:
sublist = yaml.load(sf)
except:
print "Subject list is not in proper YAML format. Please check " \
"your file"
raise Exception
# NOTE: strategies list is only needed in cpac_pipeline prep_workflow for
# creating symlinks
strategies = sorted(build_strategies(c))
# Populate subject scan map
sub_scan_map = {}
try:
for sub in sublist:
if sub['unique_id']:
s = sub['subject_id'] + "_" + sub["unique_id"]
else:
s = sub['subject_id']
scan_ids = ['scan_anat']
if 'func' in sub:
for id in sub['func']:
scan_ids.append('scan_' + str(id))
if 'rest' in sub:
for id in sub['rest']:
scan_ids.append('scan_' + str(id))
sub_scan_map[s] = scan_ids
except:
print "\n\n" + "ERROR: Subject list file not in proper format - " \
"check if you loaded the correct file?" + "\n" + \
"Error name: cpac_runner_0001" + "\n\n"
raise Exception
pipeline_timing_info = []
pipeline_timing_info.append(unique_pipeline_id)
pipeline_timing_info.append(pipeline_start_stamp)
pipeline_timing_info.append(len(sublist))
if tracking:
track_run(level='participant', participants=len(sublist))
# If we're running on cluster, execute job scheduler
if c.runOnGrid:
# Create cluster log dir
cluster_files_dir = os.path.join(c.logDirectory, 'cluster_files')
if not os.path.exists(cluster_files_dir):
os.makedirs(cluster_files_dir)
# Create strategies file
strategies_file = os.path.join(cluster_files_dir, 'strategies.pkl')
with open(strategies_file, 'w') as f:
pickle.dump(strategies, f)
# Check if its a condor job, and run that
if 'condor' in c.resourceManager.lower():
run_condor_jobs(c, config_file, strategies_file, subject_list_file,
p_name)
# All other schedulers are supported
else:
run_cpac_on_cluster(config_file, subject_list_file,
strategies_file, cluster_files_dir)
# Run on one computer
else:
if not os.path.exists(c.workingDirectory):
try:
os.makedirs(c.workingDirectory)
except:
err = "\n\n[!] CPAC says: Could not create the working " \
"directory: %s\n\nMake sure you have permissions " \
"to write to this directory.\n\n" % c.workingDirectory
raise Exception(err)
# If it only allows one, run it linearly
if c.numParticipantsAtOnce == 1:
for sub in sublist:
prep_workflow(sub, c, strategies, 1, pipeline_timing_info,
p_name, plugin, plugin_args)
return
pid = open(os.path.join(c.workingDirectory, 'pid.txt'), 'w')
# Init job queue
job_queue = []
# Allocate processes
processes = [
Process(target=prep_workflow,
args=(sub, c, strategies, 1, pipeline_timing_info, p_name,
plugin, plugin_args)) for sub in sublist
]
# If we're allocating more processes than are subjects, run them all
if len(sublist) <= c.numParticipantsAtOnce:
for p in processes:
p.start()
print >> pid, p.pid
# Otherwise manage resources to run processes incrementally
else:
idx = 0
while idx < len(sublist):
# If the job queue is empty and we haven't started indexing
if len(job_queue) == 0 and idx == 0:
# Init subject process index
idc = idx
# Launch processes (one for each subject)
for p in processes[idc:idc + c.numParticipantsAtOnce]:
p.start()
print >> pid, p.pid
job_queue.append(p)
idx += 1
# Otherwise, jobs are running - check them
else:
# Check every job in the queue's status
for job in job_queue:
# If the job is not alive
if not job.is_alive():
# Find job and delete it from queue
print 'found dead job ', job
loc = job_queue.index(job)
del job_queue[loc]
# ...and start the next available process
# (subject)
processes[idx].start()
# Append this to job queue and increment index
job_queue.append(processes[idx])
idx += 1
# Add sleep so while loop isn't consuming 100% of CPU
time.sleep(2)
# Close PID txt file to indicate finish
pid.close()
def run(config_file, subject_list_file, p_name=None, plugin=None,
plugin_args=None, tracking=True, num_subs_at_once=None, debug=False):
'''
'''
# Import packages
import commands
import os
import pickle
import time
from CPAC.pipeline.cpac_pipeline import prep_workflow
# Init variables
config_file = os.path.realpath(config_file)
subject_list_file = os.path.realpath(subject_list_file)
# take date+time stamp for run identification purposes
unique_pipeline_id = strftime("%Y%m%d%H%M%S")
pipeline_start_stamp = strftime("%Y-%m-%d_%H:%M:%S")
# Load in pipeline config file
try:
if not os.path.exists(config_file):
raise IOError
else:
c = Configuration(yaml.load(open(config_file, 'r')))
except IOError:
print "config file %s doesn't exist" % config_file
raise
except Exception as e:
raise Exception("Error reading config file - {0}\n\nError details:"
"\n{1}\n\n".format(config_file, e))
c.logDirectory = os.path.abspath(c.logDirectory)
c.workingDirectory = os.path.abspath(c.workingDirectory)
if 's3://' not in c.outputDirectory:
c.outputDirectory = os.path.abspath(c.outputDirectory)
c.crashLogDirectory = os.path.abspath(c.crashLogDirectory)
if debug:
c.write_debugging_outputs = "[1]"
if num_subs_at_once:
if not str(num_subs_at_once).isdigit():
raise Exception('[!] Value entered for --num_cores not a digit.')
c.numParticipantsAtOnce = int(num_subs_at_once)
# Do some validation
validate(c)
# Get the pipeline name
p_name = p_name or c.pipelineName
# Load in subject list
try:
with open(subject_list_file, 'r') as sf:
sublist = yaml.load(sf)
except:
print "Subject list is not in proper YAML format. Please check " \
"your file"
raise Exception
# Populate subject scan map
sub_scan_map = {}
try:
for sub in sublist:
if sub['unique_id']:
s = sub['subject_id'] + "_" + sub["unique_id"]
else:
s = sub['subject_id']
scan_ids = ['scan_anat']
if 'func' in sub:
for id in sub['func']:
scan_ids.append('scan_'+ str(id))
if 'rest' in sub:
for id in sub['rest']:
scan_ids.append('scan_'+ str(id))
sub_scan_map[s] = scan_ids
except:
print "\n\n" + "ERROR: Subject list file not in proper format - " \
"check if you loaded the correct file?" + "\n" + \
"Error name: cpac_runner_0001" + "\n\n"
raise Exception
pipeline_timing_info = []
pipeline_timing_info.append(unique_pipeline_id)
pipeline_timing_info.append(pipeline_start_stamp)
pipeline_timing_info.append(len(sublist))
if tracking:
track_run(level='participant', participants=len(sublist))
# If we're running on cluster, execute job scheduler
if c.runOnGrid:
# Create cluster log dir
cluster_files_dir = os.path.join(c.logDirectory, 'cluster_files')
if not os.path.exists(cluster_files_dir):
os.makedirs(cluster_files_dir)
# Check if its a condor job, and run that
if 'condor' in c.resourceManager.lower():
run_condor_jobs(c, config_file, subject_list_file, p_name)
# All other schedulers are supported
else:
run_cpac_on_cluster(config_file, subject_list_file, cluster_files_dir)
# Run on one computer
else:
if not os.path.exists(c.workingDirectory):
try:
os.makedirs(c.workingDirectory)
except:
err = "\n\n[!] CPAC says: Could not create the working " \
"directory: %s\n\nMake sure you have permissions " \
"to write to this directory.\n\n" % c.workingDirectory
raise Exception(err)
# If it only allows one, run it linearly
if c.numParticipantsAtOnce == 1:
for sub in sublist:
prep_workflow(sub, c, True, pipeline_timing_info,
p_name, plugin, plugin_args)
return
pid = open(os.path.join(c.workingDirectory, 'pid.txt'), 'w')
# Init job queue
job_queue = []
# Allocate processes
processes = [Process(target=prep_workflow,
args=(sub, c, True, pipeline_timing_info,
p_name, plugin, plugin_args))
for sub in sublist]
# If we're allocating more processes than are subjects, run them all
if len(sublist) <= c.numParticipantsAtOnce:
for p in processes:
p.start()
print >>pid, p.pid
# Otherwise manage resources to run processes incrementally
else:
idx = 0
while idx < len(sublist):
# If the job queue is empty and we haven't started indexing
if len(job_queue) == 0 and idx == 0:
# Init subject process index
idc = idx
# Launch processes (one for each subject)
for p in processes[idc: idc+c.numParticipantsAtOnce]:
p.start()
print >>pid, p.pid
job_queue.append(p)
idx += 1
# Otherwise, jobs are running - check them
else:
# Check every job in the queue's status
for job in job_queue:
# If the job is not alive
if not job.is_alive():
# Find job and delete it from queue
print 'found dead job ', job
loc = job_queue.index(job)
del job_queue[loc]
# ...and start the next available process
# (subject)
processes[idx].start()
# Append this to job queue and increment index
job_queue.append(processes[idx])
idx += 1
# Add sleep so while loop isn't consuming 100% of CPU
time.sleep(2)
# Close PID txt file to indicate finish
pid.close()
def run(subject_list_file, config_file=None, p_name=None, plugin=None,
plugin_args=None, tracking=True, num_subs_at_once=None, debug=False, test_config=False):
# Import packages
import commands
import os
import pickle
import time
from CPAC.pipeline.cpac_pipeline import prep_workflow
print('Run called with config file {0}'.format(config_file))
if not config_file:
import pkg_resources as p
config_file = \
p.resource_filename("CPAC",
os.path.join("resources",
"configs",
"pipeline_config_template.yml"))
# Init variables
sublist = None
config_file = os.path.realpath(config_file)
if '.yaml' in subject_list_file or '.yml' in subject_list_file:
subject_list_file = os.path.realpath(subject_list_file)
else:
from CPAC.utils.bids_utils import collect_bids_files_configs, \
bids_gen_cpac_sublist
(file_paths, config) = collect_bids_files_configs(subject_list_file,
None)
sublist = bids_gen_cpac_sublist(subject_list_file, file_paths,
config, None)
if not sublist:
import sys
print("Did not find data in {0}".format(subject_list_file))
sys.exit(1)
# take date+time stamp for run identification purposes
unique_pipeline_id = strftime("%Y%m%d%H%M%S")
pipeline_start_stamp = strftime("%Y-%m-%d_%H:%M:%S")
# Load in pipeline config file
try:
if not os.path.exists(config_file):
raise IOError
else:
c = Configuration(yaml.load(open(config_file, 'r')))
except IOError:
print "config file %s doesn't exist" % config_file
raise
except yaml.parser.ParserError as e:
error_detail = "\"%s\" at line %d" % (
e.problem,
e.problem_mark.line
)
raise Exception(
"Error parsing config file: {0}\n\n"
"Error details:\n"
" {1}"
"\n\n".format(config_file, error_detail)
)
except Exception as e:
raise Exception(
"Error parsing config file: {0}\n\n"
"Error details:\n"
" {1}"
"\n\n".format(config_file, e)
)
c.logDirectory = os.path.abspath(c.logDirectory)
c.workingDirectory = os.path.abspath(c.workingDirectory)
if 's3://' not in c.outputDirectory:
c.outputDirectory = os.path.abspath(c.outputDirectory)
c.crashLogDirectory = os.path.abspath(c.crashLogDirectory)
if debug:
c.write_debugging_outputs = "[1]"
if num_subs_at_once:
if not str(num_subs_at_once).isdigit():
raise Exception('[!] Value entered for --num_cores not a digit.')
c.numParticipantsAtOnce = int(num_subs_at_once)
# Do some validation
if not c.workingDirectory:
raise Exception('Working directory not specified')
if len(c.workingDirectory) > 70:
warnings.warn("We recommend that the working directory full path "
"should have less then 70 characters. "
"Long paths might not work in your operational system.")
warnings.warn("Current working directory: %s" % c.workingDirectory)
# Get the pipeline name
p_name = p_name or c.pipelineName
# Load in subject list
try:
if not sublist:
with open(subject_list_file, 'r') as sf:
sublist = yaml.load(sf)
except:
print "Subject list is not in proper YAML format. Please check " \
"your file"
raise Exception
# Populate subject scan map
sub_scan_map = {}
try:
for sub in sublist:
if sub['unique_id']:
s = sub['subject_id'] + "_" + sub["unique_id"]
else:
s = sub['subject_id']
scan_ids = ['scan_anat']
if 'func' in sub:
for id in sub['func']:
scan_ids.append('scan_'+ str(id))
if 'rest' in sub:
for id in sub['rest']:
scan_ids.append('scan_'+ str(id))
sub_scan_map[s] = scan_ids
except:
print "\n\n" + "ERROR: Subject list file not in proper format - " \
"check if you loaded the correct file?" + "\n" + \
"Error name: cpac_runner_0001" + "\n\n"
raise Exception
pipeline_timing_info = []
pipeline_timing_info.append(unique_pipeline_id)
pipeline_timing_info.append(pipeline_start_stamp)
pipeline_timing_info.append(len(sublist))
if tracking:
try:
track_run(level='participant', participants=len(sublist))
except:
pass
# If we're running on cluster, execute job scheduler
if c.runOnGrid:
# Create cluster log dir
cluster_files_dir = os.path.join(c.logDirectory, 'cluster_files')
if not os.path.exists(cluster_files_dir):
os.makedirs(cluster_files_dir)
# Check if its a condor job, and run that
if 'condor' in c.resourceManager.lower():
run_condor_jobs(c, config_file, subject_list_file, p_name)
# All other schedulers are supported
else:
run_cpac_on_cluster(config_file, subject_list_file, cluster_files_dir)
# Run on one computer
else:
if not os.path.exists(c.workingDirectory):
try:
os.makedirs(c.workingDirectory)
except:
err = "\n\n[!] CPAC says: Could not create the working " \
"directory: %s\n\nMake sure you have permissions " \
"to write to this directory.\n\n" % c.workingDirectory
raise Exception(err)
# If it only allows one, run it linearly
if c.numParticipantsAtOnce == 1:
for sub in sublist:
prep_workflow(sub, c, True, pipeline_timing_info,
p_name, plugin, plugin_args, test_config)
return
pid = open(os.path.join(c.workingDirectory, 'pid.txt'), 'w')
# Init job queue
job_queue = []
# Allocate processes
processes = [
Process(target=prep_workflow,
args=(sub, c, True, pipeline_timing_info,
p_name, plugin, plugin_args, test_config))
for sub in sublist
]
# If we're allocating more processes than are subjects, run them all
if len(sublist) <= c.numParticipantsAtOnce:
for p in processes:
p.start()
print >>pid, p.pid
# Otherwise manage resources to run processes incrementally
else:
idx = 0
while idx < len(sublist):
# If the job queue is empty and we haven't started indexing
if len(job_queue) == 0 and idx == 0:
# Init subject process index
idc = idx
# Launch processes (one for each subject)
for p in processes[idc: idc+c.numParticipantsAtOnce]:
p.start()
print >>pid, p.pid
job_queue.append(p)
idx += 1
# Otherwise, jobs are running - check them
else:
# Check every job in the queue's status
for job in job_queue:
# If the job is not alive
if not job.is_alive():
# Find job and delete it from queue
print 'found dead job ', job
loc = job_queue.index(job)
del job_queue[loc]
# ...and start the next available process
# (subject)
processes[idx].start()
# Append this to job queue and increment index
job_queue.append(processes[idx])
idx += 1
# Add sleep so while loop isn't consuming 100% of CPU
time.sleep(2)
# Close PID txt file to indicate finish
pid.close()
def testConfig(self, event):
'''
This function runs when the user clicks the "Test Configuration"
button in the pipeline configuration window.
It prompts the user for a sample subject list (i.e. one that they will
be using with the config they are building). Then it builds the
pipeline but does not run it. It then reports whether or not the
config will run or not depending on if the pipeline gets built
successfully.
'''
import os
import yaml
from CPAC.utils import Configuration
from CPAC.pipeline.cpac_pipeline import prep_workflow
from CPAC.pipeline.cpac_runner import build_strategies
def display(win, msg, changeBg=True):
wx.MessageBox(msg, "Error")
if changeBg:
win.SetBackgroundColour("pink")
win.SetFocus()
win.Refresh()
# Collect a sample subject list and parse it in
testDlg0 = wx.MessageDialog(
self, 'This tool will run a quick check on the current pipeline configuration.' \
' Click OK to provide a subject list you will be using with this setup.',
'Subject List',
wx.OK | wx.ICON_INFORMATION)
testDlg0.ShowModal()
testDlg0.Destroy()
dlg = wx.FileDialog(
self, message="Choose the CPAC Subject list file",
defaultDir=os.getcwd(),
defaultFile="CPAC_subject_list.yml",
wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml",
style=wx.OPEN | wx.CHANGE_DIR)
if dlg.ShowModal() == wx.ID_OK:
subListPath = dlg.GetPath()
# Load and test the subject list
sublist = yaml.load(open(os.path.realpath(subListPath), 'r'))
sub_flg = self.test_sublist(sublist)
if not sub_flg:
raise Exception
# Following code reads in the parameters and selections from the
# pipeline configuration window and populate the config_list
config_list = []
wf_counter = []
for page in self.nb.get_page_list():
switch = page.page.get_switch()
ctrl_list = page.page.get_ctrl_list()
validate = False
if switch:
switch_val = str(switch.get_selection()).lower()
if switch_val == 'on' or switch_val == 'true' or switch_val == '1':
validate = True
wf_counter.append(page.get_counter())
for ctrl in ctrl_list:
# option_name will be the selection name as it is written
# as the dictionary key of the config.yml dictionary
option_name = ctrl.get_name()
#validating
if (switch == None or validate) and ctrl.get_validation() \
and (option_name != 'derivativeList') and (option_name != 'modelConfigs'):
win = ctrl.get_ctrl()
if isinstance(ctrl.get_selection(), list):
value = ctrl.get_selection()
if not value:
display(
win, "%s field is empty or the items are not checked!" % ctrl.get_name(), False)
return
else:
value = str(ctrl.get_selection())
if len(value) == 0:
display(win, "%s field is empty!" % ctrl.get_name())
return
if '/' in value and '$' not in value and not isinstance(value, list):
if not os.path.exists(ctrl.get_selection()) and value != 'On/Off':
display(
win, "%s field contains incorrect path. Please update the path!" % ctrl.get_name())
return
config_list.append(ctrl)
# Get the user's CPAC output directory for use in this script
for config in config_list:
if config.get_name() == 'outputDirectory':
outDir = config.get_selection()
# Write out a pipeline_config file, read it in and then delete it
# (Will revise the data structure of the config files later so this
# can just pass the data structure instead of doing it this way)
try:
self.write(outDir + 'testConfig.yml', config_list)
c = Configuration(yaml.load(open(os.path.realpath(outDir + 'testConfig.yml'), 'r')))
os.remove(outDir + 'testConfig.yml')
except:
errDlg2 = wx.MessageDialog(
self, 'A problem occurred with preparing the pipeline test run. \n\n' \
'Please ensure you have rights access to the directories you' \
' have chosen for the CPAC working, crash, and output folders.',
'Test Configuration Error',
wx.OK | wx.ICON_ERROR)
errDlg2.ShowModal()
errDlg2.Destroy()
if (1 in c.runNuisance) or (c.Corrections != None):
strategies = sorted(build_strategies(c))
else:
strategies = None
# Run the actual pipeline building prep and see if it works or not
testDlg1 = wx.MessageDialog(
self, 'Click OK to run the test. This should take only a few seconds.',
'Running Test',
wx.OK | wx.ICON_INFORMATION)
testDlg1.ShowModal()
# Check file paths first
# Just getting proper names of config file parameters
try:
params_file = open(p.resource_filename('CPAC', 'GUI/resources/config_parameters.txt'), "r")
except:
print "Error: Could not open configuration parameter file.", "\n"
raise Exception
paramInfo = params_file.read().split('\n')
paramList = []
for param in paramInfo:
if param != '':
paramList.append(param.split(','))
# function for file path checking
def testFile(filepath, paramName, switch):
try:
if (1 in switch) and (filepath != None):
fileTest = open(filepath)
fileTest.close()
except:
testDlg1.Destroy()
for param in paramList:
if param[0] == paramName:
paramTitle = param[1]
paramGroup = param[2]
break
errDlgFileTest = wx.MessageDialog(
self, 'Error reading file - either it does not exist or you' \
' do not have read access. \n\n' \
'Parameter: %s \n' \
'In tab: %s \n\n' \
'Path: %s' % (paramTitle, paramGroup, filepath),
'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlgFileTest.ShowModal()
errDlgFileTest.Destroy()
testFile(c.template_brain_only_for_anat,'template_brain_only_for_anat',c.runRegistrationPreprocessing)
testFile(c.template_skull_for_anat,'template_skull_for_anat',c.runRegistrationPreprocessing)
testFile(c.PRIORS_WHITE,'PRIORS_WHITE',c.runSegmentationPreprocessing)
testFile(c.PRIORS_GRAY,'PRIORS_GRAY',c.runSegmentationPreprocessing)
testFile(c.PRIORS_CSF,'PRIORS_CSF',c.runSegmentationPreprocessing)
testFile(c.template_brain_only_for_func,'template_brain_only_for_func',c.runRegisterFuncToMNI)
testFile(c.template_skull_for_func,'template_skull_for_func',c.runRegisterFuncToMNI)
testFile(c.identityMatrix,'identityMatrix',c.runRegisterFuncToMNI)
testFile(c.boundaryBasedRegistrationSchedule,'boundaryBasedRegistrationSchedule',c.runRegisterFuncToAnat)
testFile(c.lateral_ventricles_mask,'lateral_ventricles_mask',c.runNuisance)
testFile(c.seedSpecificationFile,'seedSpecificationFile',[1])
testFile(c.roiSpecificationFile,'roiSpecificationFile',c.runROITimeseries)
testFile(c.roiSpecificationFileForSCA,'roiSpecificationFileForSCA',c.runROITimeseries)
testFile(c.maskSpecificationFile,'maskSpecificationFile',c.runVoxelTimeseries)
testFile(c.maskSpecificationFileForSCA,'maskSpecificationFileForSCA',c.runVoxelTimeseries)
testFile(c.spatialPatternMaps,'spatialPatternMaps',c.runSpatialRegression)
testFile(c.template_symmetric_brain_only,'template_symmetric_brain_only',c.runVMHC)
testFile(c.template_symmetric_skull,'template_symmetric_skull',c.runVMHC)
testFile(c.dilated_symmetric_brain_mask,'dilated_symmetric_brain_mask',c.runVMHC)
testFile(c.configFileTwomm,'configFileTwomm',c.runVMHC)
testFile(c.templateSpecificationFile,'templateSpecificationFile',c.runNetworkCentrality)
testFile(c.bascAffinityThresholdFile,'bascAffinityThresholdFile',c.runBASC)
testFile(c.cwasROIFile,'cwasROIFile',c.runCWAS)
testFile(c.cwasRegressorFile,'cwasRegressorFile',c.runCWAS)
try:
# Run the pipeline building
prep_workflow(sublist[0], c, strategies, 0)
except Exception as xxx:
print xxx
print "an exception occured"
testDlg1.Destroy()
errDlg1 = wx.MessageDialog(
self, 'There are issues with the current configuration ' \
'which need to be resolved - please check to make ' \
'sure the options you are running have the proper ' \
'pre-requisites selected.\n\nIssue Info:\n%s' % xxx,
'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlg1.ShowModal()
errDlg1.Destroy()
else:
testDlg1.Destroy()
okDlg1 = wx.MessageDialog(
self, 'The current configuration will run successfully. You can safely' \
' save and run this setup!',
'Pipeline Ready',
wx.OK | wx.ICON_INFORMATION)
okDlg1.ShowModal()
okDlg1.Destroy()
def testConfig(self, event):
'''
This function runs when the user clicks the "Test Configuration"
button in the pipeline configuration window.
It prompts the user for a sample subject list (i.e. one that they will
be using with the config they are building). Then it builds the
pipeline but does not run it. It then reports whether or not the
config will run or not depending on if the pipeline gets built
successfully.
'''
# Import packages
import os
import yaml
from CPAC.utils import Configuration
from CPAC.pipeline.cpac_pipeline import prep_workflow
from CPAC.pipeline.cpac_runner import build_strategies
def display(win, msg, changeBg=True):
wx.MessageBox(msg, "Error")
if changeBg:
win.SetBackgroundColour("pink")
win.SetFocus()
win.Refresh()
# Collect a sample subject list and parse it in
testDlg0 = wx.MessageDialog(
self, 'This tool will run a quick check on the current pipeline '\
'configuration. Click OK to provide a subject list you ' \
'will be using with this setup.',
'Subject List',
wx.OK | wx.ICON_INFORMATION)
testDlg0.ShowModal()
testDlg0.Destroy()
dlg = wx.FileDialog(self,
message="Choose the CPAC Subject list file",
defaultDir=os.getcwd(),
defaultFile="CPAC_subject_list.yml",
wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml",
style=wx.OPEN | wx.CHANGE_DIR)
if dlg.ShowModal() == wx.ID_OK:
subListPath = dlg.GetPath()
# Load and test the subject list
print 'Checking subject list: %s...' % subListPath
sublist = yaml.load(open(os.path.realpath(subListPath), 'r'))
sub_flg = self.test_sublist(sublist)
if not sub_flg:
raise Exception
print 'Subject list looks good!'
# Following code reads in the parameters and selections from the
# pipeline configuration window and populate the config_list
config_list = []
wf_counter = []
for page in self.nb.get_page_list():
switch = page.page.get_switch()
ctrl_list = page.page.get_ctrl_list()
validate = False
if switch:
switch_val = str(switch.get_selection()).lower()
if switch_val == 'on' or switch_val == 'true' or \
switch_val == '1':
validate = True
wf_counter.append(page.get_counter())
for ctrl in ctrl_list:
# option_name will be the selection name as it is written
# as the dictionary key of the config.yml dictionary
option_name = ctrl.get_name()
#validating
if (switch == None or validate) and ctrl.get_validation() \
and (option_name != 'derivativeList') and \
(option_name != 'modelConfigs'):
win = ctrl.get_ctrl()
if isinstance(ctrl.get_selection(), list):
value = ctrl.get_selection()
if not value:
display(
win, "%s field is empty or the items are " \
"not checked!" % ctrl.get_name(), False)
return
elif (option_name == "tsa_roi_paths") or \
(option_name == "sca_roi_paths"):
# fires if the control is the checkbox grid for
# multiple paths assigned to multiple options
# (i.e. timeseries analysis)
config_list.append(ctrl)
continue
else:
value = str(ctrl.get_selection())
if len(value) == 0:
display(win, "%s field is empty!" % ctrl.get_name())
return
if '/' in value and '$' not in value and not \
isinstance(value, list):
if not os.path.exists(ctrl.get_selection()) and \
value != 'On/Off':
display(
win, "%s field contains incorrect path. " \
"Please update the path!" % ctrl.get_name())
return
config_list.append(ctrl)
# Write out a pipeline_config file, read it in and then delete it
# (Will revise the data structure of the config files later so this
# can just pass the data structure instead of doing it this way)
try:
test_cfg_yml = '/tmp/test_config.yml'
self.write(test_cfg_yml, config_list)
c = Configuration(
yaml.load(open(os.path.realpath(test_cfg_yml), 'r')))
os.remove(test_cfg_yml)
except:
errDlg2 = wx.MessageDialog(
self, 'A problem occurred with preparing the pipeline test run. \n\n' \
'Please ensure you have rights access to the directories you' \
' have chosen for the CPAC working, crash, and output folders.',
'Test Configuration Error',
wx.OK | wx.ICON_ERROR)
errDlg2.ShowModal()
errDlg2.Destroy()
if (1 in c.runNuisance) or (c.Regressors != None):
strategies = sorted(build_strategies(c))
else:
strategies = None
# Run the actual pipeline building prep and see if it works or not
testDlg1 = wx.MessageDialog(
self,
'Click OK to run the test. This should take only a few seconds.',
'Running Test', wx.OK | wx.ICON_INFORMATION)
testDlg1.ShowModal()
# Check file paths first
# Just getting proper names of config file parameters
try:
params_file = open(
p.resource_filename('CPAC',
'GUI/resources/config_parameters.txt'),
"r")
except:
print "Error: Could not open configuration parameter file.", "\n"
raise Exception
paramInfo = params_file.read().split('\n')
paramList = []
for param in paramInfo:
if param != '':
paramList.append(param.split(','))
# function for file path checking
def testFile(filepath, paramName, switch):
try:
if (1 in switch) and (filepath != None):
fileTest = open(filepath)
fileTest.close()
except:
testDlg1.Destroy()
for param in paramList:
if param[0] == paramName:
paramTitle = param[1]
paramGroup = param[2]
break
errDlgFileTest = wx.MessageDialog(
self, 'Error reading file - either it does not exist or '\
'you do not have read access. \n\n' \
'Parameter: %s \n' \
'In tab: %s \n\n' \
'Path: %s' % (paramTitle, paramGroup, filepath),
'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlgFileTest.ShowModal()
errDlgFileTest.Destroy()
# Check S3 output bucket access if writing to S3
output_dir = c.outputDirectory
s3_str = 's3://'
if output_dir.lower().startswith(s3_str):
output_dir_sp = output_dir.split('/')
output_dir_sp[0] = output_dir_sp[0].lower()
output_dir = '/'.join(output_dir_sp)
if type(output_dir) is str and output_dir.lower().startswith(s3_str):
from indi_aws import fetch_creds
creds_path = c.awsOutputBucketCredentials
bucket_name = output_dir.split(s3_str)[1].split('/')[0]
try:
bucket = fetch_creds.return_bucket(creds_path, bucket_name)
print 'Connection with output bucket "%s" successful!' % bucket_name
except Exception as exc:
err_msg = 'Unable to access output S3 bucket: "%s" with '\
'credentials in: "%s". Check bucket name '\
'and credentials file and try again'\
% (bucket_name, creds_path)
testDlg1.Destroy()
errDlg1 = wx.MessageDialog(self, err_msg, 'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlg1.ShowModal()
errDlg1.Destroy()
return
testFile(c.template_brain_only_for_anat, \
'template_brain_only_for_anat',[1])
testFile(c.template_skull_for_anat, 'template_skull_for_anat', [1])
testFile(c.PRIORS_WHITE, 'PRIORS_WHITE',
c.runSegmentationPreprocessing)
testFile(c.PRIORS_GRAY, 'PRIORS_GRAY', c.runSegmentationPreprocessing)
testFile(c.PRIORS_CSF, 'PRIORS_CSF', c.runSegmentationPreprocessing)
testFile(c.template_brain_only_for_func, \
'template_brain_only_for_func',c.runRegisterFuncToMNI)
testFile(c.template_skull_for_func,'template_skull_for_func', \
c.runRegisterFuncToMNI)
testFile(c.identityMatrix, 'identityMatrix', c.runRegisterFuncToMNI)
testFile(c.boundaryBasedRegistrationSchedule, \
'boundaryBasedRegistrationSchedule', \
c.runRegisterFuncToAnat)
testFile(c.lateral_ventricles_mask,'lateral_ventricles_mask', \
c.runNuisance)
testFile(c.template_symmetric_brain_only, \
'template_symmetric_brain_only',c.runVMHC)
testFile(c.template_symmetric_skull,'template_symmetric_skull', \
c.runVMHC)
testFile(c.dilated_symmetric_brain_mask, \
'dilated_symmetric_brain_mask',c.runVMHC)
testFile(c.configFileTwomm, 'configFileTwomm', c.runVMHC)
testFile(c.templateSpecificationFile,'templateSpecificationFile', \
c.runNetworkCentrality)
if c.tsa_roi_paths and type(c.tsa_roi_paths[0]) == dict:
for roi_path in c.tsa_roi_paths[0].keys():
testFile(roi_path, "tsa_roi_paths", c.runROITimeseries)
if c.sca_roi_paths and type(c.sca_roi_paths[0]) == dict:
for roi_path in c.sca_roi_paths[0].keys():
testFile(roi_path, "sca_roi_paths", c.runSCA)
try:
# Run the pipeline building
prep_workflow(sublist[0], c, strategies, 0)
except Exception as xxx:
print xxx
print "an exception occurred"
testDlg1.Destroy()
errDlg1 = wx.MessageDialog(
self, 'There are issues with the current configuration ' \
'which need to be resolved - please check to make ' \
'sure the options you are running have the proper ' \
'pre-requisites selected.\n\nIssue Info:\n%s' \
% str(xxx),
'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlg1.ShowModal()
errDlg1.Destroy()
else:
testDlg1.Destroy()
okDlg1 = wx.MessageDialog(
self, 'The current configuration will run successfully. You '\
'can safely save and run this setup!',
'Pipeline Ready',
wx.OK | wx.ICON_INFORMATION)
okDlg1.ShowModal()
okDlg1.Destroy()
def testConfig(self, event):
'''
This function runs when the user clicks the "Test Configuration"
button in the pipeline configuration window.
It prompts the user for a sample subject list (i.e. one that they will
be using with the config they are building). Then it builds the
pipeline but does not run it. It then reports whether or not the
config will run or not depending on if the pipeline gets built
successfully.
'''
# Import packages
import os
import yaml
from CPAC.utils import Configuration
from CPAC.pipeline.cpac_pipeline import prep_workflow
from CPAC.pipeline.cpac_runner import build_strategies
def display(win, msg, changeBg=True):
wx.MessageBox(msg, "Error")
if changeBg:
win.SetBackgroundColour("pink")
win.SetFocus()
win.Refresh()
# Collect a sample subject list and parse it in
testDlg0 = wx.MessageDialog(
self, 'This tool will run a quick check on the current pipeline '
'configuration. Click OK to provide a subject list you '
'will be using with this setup.',
'Subject List',
wx.OK | wx.ICON_INFORMATION)
testDlg0.ShowModal()
testDlg0.Destroy()
dlg = wx.FileDialog(
self, message="Choose the CPAC Subject list file",
defaultDir=os.getcwd(),
defaultFile="CPAC_subject_list.yml",
wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml",
style=wx.OPEN | wx.CHANGE_DIR)
if dlg.ShowModal() == wx.ID_OK:
subListPath = dlg.GetPath()
# Load and test the subject list
print 'Checking subject list: %s...' % subListPath
sublist = yaml.load(open(os.path.realpath(subListPath), 'r'))
sub_flg = self.test_sublist(sublist)
if not sub_flg:
raise Exception
print 'Subject list looks good!'
# Following code reads in the parameters and selections from the
# pipeline configuration window and populate the config_list
config_list = []
wf_counter = []
for page in self.nb.get_page_list():
switch = page.page.get_switch()
ctrl_list = page.page.get_ctrl_list()
validate = False
if switch:
switch_val = str(switch.get_selection()).lower()
if switch_val == 'on' or switch_val == 'true' or \
switch_val == '1':
validate = True
wf_counter.append(page.get_counter())
for ctrl in ctrl_list:
# option_name will be the selection name as it is written
# as the dictionary key of the config.yml dictionary
option_name = ctrl.get_name()
#validating
if (switch == None or validate) and ctrl.get_validation() \
and (option_name != 'derivativeList') and \
(option_name != 'modelConfigs'):
win = ctrl.get_ctrl()
if isinstance(ctrl.get_selection(), list):
value = ctrl.get_selection()
if not value:
display(
win, "%s field is empty or the items are " \
"not checked!" % ctrl.get_name(), False)
return
elif (option_name == "tsa_roi_paths") or \
(option_name == "sca_roi_paths"):
# fires if the control is the checkbox grid for
# multiple paths assigned to multiple options
# (i.e. timeseries analysis)
config_list.append(ctrl)
continue
else:
value = str(ctrl.get_selection())
if len(value) == 0:
display(win, "%s field is empty!" % ctrl.get_name())
return
if '/' in value and '$' not in value and not \
isinstance(value, list):
if not os.path.exists(ctrl.get_selection()) and \
value != 'On/Off':
display(
win, "%s field contains incorrect path. " \
"Please update the path!" % ctrl.get_name())
return
config_list.append(ctrl)
# Write out a pipeline_config file, read it in and then delete it
# (Will revise the data structure of the config files later so this
# can just pass the data structure instead of doing it this way)
try:
test_cfg_yml = '/tmp/test_config.yml'
self.write(test_cfg_yml, config_list)
c = Configuration(yaml.load(open(os.path.realpath(test_cfg_yml), 'r')))
os.remove(test_cfg_yml)
except:
errDlg2 = wx.MessageDialog(
self, 'A problem occurred with preparing the pipeline test run. \n\n' \
'Please ensure you have rights access to the directories you' \
' have chosen for the CPAC working, crash, and output folders.',
'Test Configuration Error',
wx.OK | wx.ICON_ERROR)
errDlg2.ShowModal()
errDlg2.Destroy()
if (1 in c.runNuisance) or (c.Regressors != None):
strategies = sorted(build_strategies(c))
else:
strategies = None
# Run the actual pipeline building prep and see if it works or not
testDlg1 = wx.MessageDialog(
self, 'Click OK to run the test. This should take only a few seconds.',
'Running Test',
wx.OK | wx.ICON_INFORMATION)
testDlg1.ShowModal()
# Check file paths first
# Just getting proper names of config file parameters
try:
params_file = open(p.resource_filename('CPAC', 'GUI/resources/config_parameters.txt'), "r")
except:
print "Error: Could not open configuration parameter file.", "\n"
raise Exception
paramInfo = params_file.read().split('\n')
paramList = []
for param in paramInfo:
if param != '':
paramList.append(param.split(','))
# function for file path checking
def testFile(filepath, paramName, switch):
try:
if (1 in switch) and (filepath != None):
fileTest = open(filepath)
fileTest.close()
except:
testDlg1.Destroy()
for param in paramList:
if param[0] == paramName:
paramTitle = param[1]
paramGroup = param[2]
break
errDlgFileTest = wx.MessageDialog(
self, 'Error reading file - either it does not exist or '\
'you do not have read access. \n\n' \
'Parameter: %s \n' \
'In tab: %s \n\n' \
'Path: %s' % (paramTitle, paramGroup, filepath),
'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlgFileTest.ShowModal()
errDlgFileTest.Destroy()
# Check S3 output bucket access if writing to S3
output_dir = c.outputDirectory
s3_str = 's3://'
if output_dir.lower().startswith(s3_str):
output_dir_sp = output_dir.split('/')
output_dir_sp[0] = output_dir_sp[0].lower()
output_dir = '/'.join(output_dir_sp)
if type(output_dir) is str and output_dir.lower().startswith(s3_str):
from indi_aws import fetch_creds
creds_path = c.awsOutputBucketCredentials
bucket_name = output_dir.split(s3_str)[1].split('/')[0]
try:
bucket = fetch_creds.return_bucket(creds_path, bucket_name)
print 'Connection with output bucket "%s" successful!' % bucket_name
except Exception as exc:
err_msg = 'Unable to access output S3 bucket: "%s" with '\
'credentials in: "%s". Check bucket name '\
'and credentials file and try again'\
% (bucket_name, creds_path)
testDlg1.Destroy()
errDlg1 = wx.MessageDialog(self, err_msg, 'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlg1.ShowModal()
errDlg1.Destroy()
return
testFile(c.template_brain_only_for_anat, \
'template_brain_only_for_anat',[1])
testFile(c.template_skull_for_anat,'template_skull_for_anat',[1])
testFile(c.PRIORS_WHITE,'PRIORS_WHITE',c.runSegmentationPreprocessing)
testFile(c.PRIORS_GRAY,'PRIORS_GRAY',c.runSegmentationPreprocessing)
testFile(c.PRIORS_CSF,'PRIORS_CSF',c.runSegmentationPreprocessing)
testFile(c.template_brain_only_for_func, \
'template_brain_only_for_func',c.runRegisterFuncToMNI)
testFile(c.template_skull_for_func,'template_skull_for_func', \
c.runRegisterFuncToMNI)
testFile(c.identityMatrix,'identityMatrix',c.runRegisterFuncToMNI)
testFile(c.boundaryBasedRegistrationSchedule, \
'boundaryBasedRegistrationSchedule', \
c.runRegisterFuncToAnat)
testFile(c.lateral_ventricles_mask,'lateral_ventricles_mask', \
c.runNuisance)
testFile(c.template_symmetric_brain_only, \
'template_symmetric_brain_only',c.runVMHC)
testFile(c.template_symmetric_skull,'template_symmetric_skull', \
c.runVMHC)
testFile(c.dilated_symmetric_brain_mask, \
'dilated_symmetric_brain_mask',c.runVMHC)
testFile(c.configFileTwomm,'configFileTwomm',c.runVMHC)
testFile(c.templateSpecificationFile,'templateSpecificationFile', \
c.runNetworkCentrality)
if c.tsa_roi_paths and type(c.tsa_roi_paths[0]) == dict:
for roi_path in c.tsa_roi_paths[0].keys():
testFile(roi_path, "tsa_roi_paths", c.runROITimeseries)
if c.sca_roi_paths and type(c.sca_roi_paths[0]) == dict:
for roi_path in c.sca_roi_paths[0].keys():
testFile(roi_path, "sca_roi_paths", c.runSCA)
try:
# Run the pipeline building
prep_workflow(sublist[0], c, strategies, 0)
except Exception as xxx:
print xxx
print "an exception occurred"
testDlg1.Destroy()
errDlg1 = wx.MessageDialog(
self, 'There are issues with the current configuration ' \
'which need to be resolved - please check to make ' \
'sure the options you are running have the proper ' \
'pre-requisites selected.\n\nIssue Info:\n%s' \
% str(xxx),
'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlg1.ShowModal()
errDlg1.Destroy()
else:
testDlg1.Destroy()
okDlg1 = wx.MessageDialog(
self, 'The current configuration will run successfully. You '\
'can safely save and run this setup!',
'Pipeline Ready',
wx.OK | wx.ICON_INFORMATION)
okDlg1.ShowModal()
okDlg1.Destroy()
def testConfig(self, event):
'''
This function runs when the user clicks the "Test Configuration" button in the
pipeline configuration window.
It prompts the user for a sample subject list (i.e. one that they will be using
with the config they are building). Then it builds the pipeline but does not
run it. It then reports whether or not the config will run or not depending
on if the pipeline gets built successfully.
'''
import os
import yaml
from CPAC.utils import Configuration
from CPAC.pipeline.cpac_pipeline import prep_workflow
from CPAC.pipeline.cpac_runner import build_strategies
def display(win, msg, changeBg=True):
wx.MessageBox(msg, "Error")
if changeBg:
win.SetBackgroundColour("pink")
win.SetFocus()
win.Refresh()
# Collect a sample subject list and parse it in
testDlg0 = wx.MessageDialog(
self, 'This tool will run a quick check on the current pipeline configuration.' \
' Click OK to provide a subject list you will be using with this setup.',
'Subject List',
wx.OK | wx.ICON_INFORMATION)
testDlg0.ShowModal()
testDlg0.Destroy()
dlg = wx.FileDialog(self,
message="Choose the CPAC Subject list file",
defaultDir=os.getcwd(),
defaultFile="CPAC_subject_list.yml",
wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml",
style=wx.OPEN | wx.CHANGE_DIR)
if dlg.ShowModal() == wx.ID_OK:
subListPath = dlg.GetPath()
sublist = yaml.load(open(os.path.realpath(subListPath), 'r'))
# Check to ensure the user is providing an actual subject
# list and not some other kind of file
try:
subInfo = sublist[0]
except:
errDlg4 = wx.MessageDialog(
self, 'ERROR: Subject list file not in proper format - check if you' \
' loaded the correct file? \n\n' \
'Error name: config_window_0001',
'Subject List Error',
wx.OK | wx.ICON_ERROR)
errDlg4.ShowModal()
errDlg4.Destroy()
raise Exception
# Another check to ensure the actual subject list was generated
# properly and that it will work
if 'subject_id' not in subInfo:
errDlg3 = wx.MessageDialog(
self, 'ERROR: Subject list file not in proper format - check if you' \
' loaded the correct file? \n\n' \
'Error name: config_window_0002',
'Subject List Error',
wx.OK | wx.ICON_ERROR)
errDlg3.ShowModal()
errDlg3.Destroy()
raise Exception
# Following code reads in the parameters and selections from the
# pipeline configuration window and populate the config_list
config_list = []
wf_counter = []
#print "self.nb.get_page_list()", self.nb.get_page_list()
for page in self.nb.get_page_list():
#print "page ----> ", page
switch = page.page.get_switch()
#print "switch ---->", switch
ctrl_list = page.page.get_ctrl_list()
validate = False
if switch:
switch_val = str(switch.get_selection()).lower()
#print "switch_val ---->", switch_val
if switch_val == 'on' or switch_val == 'true' or switch_val == '1':
validate = True
wf_counter.append(page.get_counter())
for ctrl in ctrl_list:
#validating
if (switch == None or validate) and ctrl.get_validation():
win = ctrl.get_ctrl()
#print "validating ctrl-->", ctrl.get_name()
#print "ctrl.get_selection()", ctrl.get_selection()
#print "type(ctrl.get_selection())", type(ctrl.get_selection())
if isinstance(ctrl.get_selection(), list):
value = ctrl.get_selection()
if not value:
display(
win,
"%s field is empty or the items are not checked!"
% ctrl.get_name(), False)
return
else:
value = str(ctrl.get_selection())
if len(value) == 0:
display(win, "%s field is empty!" % ctrl.get_name())
return
if '/' in value and '$' not in value and not isinstance(
value, list):
if not os.path.exists(ctrl.get_selection()):
display(
win,
"%s field contains incorrect path. Please update the path!"
% ctrl.get_name())
return
config_list.append(ctrl)
# Get the user's CPAC output directory for use in this script
for config in config_list:
#print config.get_name(), " ", config.get_selection()
if config.get_name() == 'outputDirectory':
outDir = config.get_selection()
# Write out a pipeline_config file, read it in and then delete it
# (Will revise the data structure of the config files later so this
# can just pass the data structure instead of doing it this way)
try:
self.write(outDir + 'testConfig.yml', config_list)
c = Configuration(
yaml.load(
open(os.path.realpath(outDir + 'testConfig.yml'), 'r')))
os.remove(outDir + 'testConfig.yml')
except:
errDlg2 = wx.MessageDialog(
self, 'A problem occurred with preparing the pipeline test run. \n\n' \
'Please ensure you have rights access to the directories you' \
' have chosen for the CPAC working, crash, and output folders.',
'Test Configuration Error',
wx.OK | wx.ICON_ERROR)
errDlg2.ShowModal()
errDlg2.Destroy()
if (1 in c.runNuisance) or (c.Corrections != None):
strategies = sorted(build_strategies(c))
else:
strategies = None
# Run the actual pipeline building prep and see if it works or not
try:
testDlg1 = wx.MessageDialog(
self,
'Click OK to run the test. This should take only a few seconds.',
'Running Test', wx.OK | wx.ICON_INFORMATION)
testDlg1.ShowModal()
prep_workflow(sublist[0], c, strategies, 0)
except:
testDlg1.Destroy()
errDlg1 = wx.MessageDialog(
self, 'There are issues with the current configuration which need to be' \
' resolved - please check to make sure the options you are running' \
' have the proper pre-requisites selected.',
'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlg1.ShowModal()
errDlg1.Destroy()
else:
testDlg1.Destroy()
okDlg1 = wx.MessageDialog(
self, 'The current configuration will run successfully. You can safely' \
' save and run this setup!',
'Pipeline Ready',
wx.OK | wx.ICON_INFORMATION)
okDlg1.ShowModal()
okDlg1.Destroy()
def testConfig(self, event):
'''
This function runs when the user clicks the "Test Configuration"
button in the pipeline configuration window.
It prompts the user for a sample subject list (i.e. one that they will
be using with the config they are building). Then it builds the
pipeline but does not run it. It then reports whether or not the
config will run or not depending on if the pipeline gets built
successfully.
'''
import os
import yaml
from CPAC.utils import Configuration
from CPAC.pipeline.cpac_pipeline import prep_workflow
from CPAC.pipeline.cpac_runner import build_strategies
def display(win, msg, changeBg=True):
wx.MessageBox(msg, "Error")
if changeBg:
win.SetBackgroundColour("pink")
win.SetFocus()
win.Refresh()
# Collect a sample subject list and parse it in
testDlg0 = wx.MessageDialog(
self, 'This tool will run a quick check on the current pipeline configuration.' \
' Click OK to provide a subject list you will be using with this setup.',
'Subject List',
wx.OK | wx.ICON_INFORMATION)
testDlg0.ShowModal()
testDlg0.Destroy()
dlg = wx.FileDialog(self,
message="Choose the CPAC Subject list file",
defaultDir=os.getcwd(),
defaultFile="CPAC_subject_list.yml",
wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml",
style=wx.OPEN | wx.CHANGE_DIR)
if dlg.ShowModal() == wx.ID_OK:
subListPath = dlg.GetPath()
# Load and test the subject list
sublist = yaml.load(open(os.path.realpath(subListPath), 'r'))
sub_flg = self.test_sublist(sublist)
if not sub_flg:
raise Exception
# Following code reads in the parameters and selections from the
# pipeline configuration window and populate the config_list
config_list = []
wf_counter = []
for page in self.nb.get_page_list():
switch = page.page.get_switch()
ctrl_list = page.page.get_ctrl_list()
validate = False
if switch:
switch_val = str(switch.get_selection()).lower()
if switch_val == 'on' or switch_val == 'true' or switch_val == '1':
validate = True
wf_counter.append(page.get_counter())
for ctrl in ctrl_list:
# option_name will be the selection name as it is written
# as the dictionary key of the config.yml dictionary
option_name = ctrl.get_name()
#validating
if (switch == None or validate) and ctrl.get_validation() \
and (option_name != 'derivativeList') and (option_name != 'modelConfigs'):
win = ctrl.get_ctrl()
if isinstance(ctrl.get_selection(), list):
value = ctrl.get_selection()
if not value:
display(
win,
"%s field is empty or the items are not checked!"
% ctrl.get_name(), False)
return
else:
value = str(ctrl.get_selection())
if len(value) == 0:
display(win, "%s field is empty!" % ctrl.get_name())
return
if '/' in value and '$' not in value and not isinstance(
value, list):
if not os.path.exists(
ctrl.get_selection()) and value != 'On/Off':
display(
win,
"%s field contains incorrect path. Please update the path!"
% ctrl.get_name())
return
config_list.append(ctrl)
# Get the user's CPAC output directory for use in this script
for config in config_list:
if config.get_name() == 'outputDirectory':
outDir = config.get_selection()
# Write out a pipeline_config file, read it in and then delete it
# (Will revise the data structure of the config files later so this
# can just pass the data structure instead of doing it this way)
try:
self.write(outDir + 'testConfig.yml', config_list)
c = Configuration(
yaml.load(
open(os.path.realpath(outDir + 'testConfig.yml'), 'r')))
os.remove(outDir + 'testConfig.yml')
except:
errDlg2 = wx.MessageDialog(
self, 'A problem occurred with preparing the pipeline test run. \n\n' \
'Please ensure you have rights access to the directories you' \
' have chosen for the CPAC working, crash, and output folders.',
'Test Configuration Error',
wx.OK | wx.ICON_ERROR)
errDlg2.ShowModal()
errDlg2.Destroy()
if (1 in c.runNuisance) or (c.Corrections != None):
strategies = sorted(build_strategies(c))
else:
strategies = None
# Run the actual pipeline building prep and see if it works or not
testDlg1 = wx.MessageDialog(
self,
'Click OK to run the test. This should take only a few seconds.',
'Running Test', wx.OK | wx.ICON_INFORMATION)
testDlg1.ShowModal()
# Check file paths first
# Just getting proper names of config file parameters
try:
params_file = open(
p.resource_filename('CPAC',
'GUI/resources/config_parameters.txt'),
"r")
except:
print "Error: Could not open configuration parameter file.", "\n"
raise Exception
paramInfo = params_file.read().split('\n')
paramList = []
for param in paramInfo:
if param != '':
paramList.append(param.split(','))
# function for file path checking
def testFile(filepath, paramName, switch):
try:
if (1 in switch) and (filepath != None):
fileTest = open(filepath)
fileTest.close()
except:
testDlg1.Destroy()
for param in paramList:
if param[0] == paramName:
paramTitle = param[1]
paramGroup = param[2]
break
errDlgFileTest = wx.MessageDialog(
self, 'Error reading file - either it does not exist or you' \
' do not have read access. \n\n' \
'Parameter: %s \n' \
'In tab: %s \n\n' \
'Path: %s' % (paramTitle, paramGroup, filepath),
'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlgFileTest.ShowModal()
errDlgFileTest.Destroy()
testFile(c.template_brain_only_for_anat,
'template_brain_only_for_anat',
c.runRegistrationPreprocessing)
testFile(c.template_skull_for_anat, 'template_skull_for_anat',
c.runRegistrationPreprocessing)
testFile(c.PRIORS_WHITE, 'PRIORS_WHITE',
c.runSegmentationPreprocessing)
testFile(c.PRIORS_GRAY, 'PRIORS_GRAY', c.runSegmentationPreprocessing)
testFile(c.PRIORS_CSF, 'PRIORS_CSF', c.runSegmentationPreprocessing)
testFile(c.template_brain_only_for_func,
'template_brain_only_for_func', c.runRegisterFuncToMNI)
testFile(c.template_skull_for_func, 'template_skull_for_func',
c.runRegisterFuncToMNI)
testFile(c.identityMatrix, 'identityMatrix', c.runRegisterFuncToMNI)
testFile(c.boundaryBasedRegistrationSchedule,
'boundaryBasedRegistrationSchedule', c.runRegisterFuncToAnat)
testFile(c.lateral_ventricles_mask, 'lateral_ventricles_mask',
c.runNuisance)
testFile(c.seedSpecificationFile, 'seedSpecificationFile', [1])
testFile(c.roiSpecificationFile, 'roiSpecificationFile',
c.runROITimeseries)
testFile(c.roiSpecificationFileForSCA, 'roiSpecificationFileForSCA',
c.runROITimeseries)
testFile(c.maskSpecificationFile, 'maskSpecificationFile',
c.runVoxelTimeseries)
testFile(c.maskSpecificationFileForSCA, 'maskSpecificationFileForSCA',
c.runVoxelTimeseries)
testFile(c.spatialPatternMaps, 'spatialPatternMaps',
c.runSpatialRegression)
testFile(c.template_symmetric_brain_only,
'template_symmetric_brain_only', c.runVMHC)
testFile(c.template_symmetric_skull, 'template_symmetric_skull',
c.runVMHC)
testFile(c.dilated_symmetric_brain_mask,
'dilated_symmetric_brain_mask', c.runVMHC)
testFile(c.configFileTwomm, 'configFileTwomm', c.runVMHC)
testFile(c.templateSpecificationFile, 'templateSpecificationFile',
c.runNetworkCentrality)
testFile(c.bascAffinityThresholdFile, 'bascAffinityThresholdFile',
c.runBASC)
testFile(c.cwasROIFile, 'cwasROIFile', c.runCWAS)
testFile(c.cwasRegressorFile, 'cwasRegressorFile', c.runCWAS)
try:
# Run the pipeline building
prep_workflow(sublist[0], c, strategies, 0)
except Exception as xxx:
print xxx
print "an exception occured"
testDlg1.Destroy()
errDlg1 = wx.MessageDialog(
self, 'There are issues with the current configuration ' \
'which need to be resolved - please check to make ' \
'sure the options you are running have the proper ' \
'pre-requisites selected.\n\nIssue Info:\n%s' % xxx,
'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlg1.ShowModal()
errDlg1.Destroy()
else:
testDlg1.Destroy()
okDlg1 = wx.MessageDialog(
self, 'The current configuration will run successfully. You can safely' \
' save and run this setup!',
'Pipeline Ready',
wx.OK | wx.ICON_INFORMATION)
okDlg1.ShowModal()
okDlg1.Destroy()
def testConfig(self, event):
'''
This function runs when the user clicks the "Test Configuration" button in the
pipeline configuration window.
It prompts the user for a sample subject list (i.e. one that they will be using
with the config they are building). Then it builds the pipeline but does not
run it. It then reports whether or not the config will run or not depending
on if the pipeline gets built successfully.
'''
import os
import yaml
from CPAC.utils import Configuration
from CPAC.pipeline.cpac_pipeline import prep_workflow
from CPAC.pipeline.cpac_runner import build_strategies
def display(win, msg, changeBg=True):
wx.MessageBox(msg, "Error")
if changeBg:
win.SetBackgroundColour("pink")
win.SetFocus()
win.Refresh()
# Collect a sample subject list and parse it in
testDlg0 = wx.MessageDialog(
self, 'This tool will run a quick check on the current pipeline configuration.' \
' Click OK to provide a subject list you will be using with this setup.',
'Subject List',
wx.OK | wx.ICON_INFORMATION)
testDlg0.ShowModal()
testDlg0.Destroy()
dlg = wx.FileDialog(
self, message="Choose the CPAC Subject list file",
defaultDir=os.getcwd(),
defaultFile="CPAC_subject_list.yml",
wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml",
style=wx.OPEN | wx.CHANGE_DIR)
if dlg.ShowModal() == wx.ID_OK:
subListPath = dlg.GetPath()
sublist = yaml.load(open(os.path.realpath(subListPath), 'r'))
# Check to ensure the user is providing an actual subject
# list and not some other kind of file
try:
subInfo = sublist[0]
except:
errDlg4 = wx.MessageDialog(
self, 'ERROR: Subject list file not in proper format - check if you' \
' loaded the correct file? \n\n' \
'Error name: config_window_0001',
'Subject List Error',
wx.OK | wx.ICON_ERROR)
errDlg4.ShowModal()
errDlg4.Destroy()
raise Exception
# Another check to ensure the actual subject list was generated
# properly and that it will work
if 'subject_id' not in subInfo:
errDlg3 = wx.MessageDialog(
self, 'ERROR: Subject list file not in proper format - check if you' \
' loaded the correct file? \n\n' \
'Error name: config_window_0002',
'Subject List Error',
wx.OK | wx.ICON_ERROR)
errDlg3.ShowModal()
errDlg3.Destroy()
raise Exception
# Following code reads in the parameters and selections from the
# pipeline configuration window and populate the config_list
config_list = []
wf_counter = []
#print "self.nb.get_page_list()", self.nb.get_page_list()
for page in self.nb.get_page_list():
#print "page ----> ", page
switch = page.page.get_switch()
#print "switch ---->", switch
ctrl_list = page.page.get_ctrl_list()
validate = False
if switch:
switch_val = str(switch.get_selection()).lower()
#print "switch_val ---->", switch_val
if switch_val == 'on' or switch_val == 'true' or switch_val == '1':
validate = True
wf_counter.append(page.get_counter())
for ctrl in ctrl_list:
#validating
if (switch == None or validate) and ctrl.get_validation():
win = ctrl.get_ctrl()
#print "validating ctrl-->", ctrl.get_name()
#print "ctrl.get_selection()", ctrl.get_selection()
#print "type(ctrl.get_selection())", type(ctrl.get_selection())
if isinstance(ctrl.get_selection(), list):
value = ctrl.get_selection()
if not value:
display(
win, "%s field is empty or the items are not checked!" % ctrl.get_name(), False)
return
else:
value = str(ctrl.get_selection())
if len(value) == 0:
display(win, "%s field is empty!" % ctrl.get_name())
return
if '/' in value and '$' not in value and not isinstance(value, list):
if not os.path.exists(ctrl.get_selection()):
display(
win, "%s field contains incorrect path. Please update the path!" % ctrl.get_name())
return
config_list.append(ctrl)
# Get the user's CPAC output directory for use in this script
for config in config_list:
#print config.get_name(), " ", config.get_selection()
if config.get_name() == 'outputDirectory':
outDir = config.get_selection()
# Write out a pipeline_config file, read it in and then delete it
# (Will revise the data structure of the config files later so this
# can just pass the data structure instead of doing it this way)
try:
self.write(outDir + 'testConfig.yml', config_list)
c = Configuration(yaml.load(open(os.path.realpath(outDir + 'testConfig.yml'), 'r')))
os.remove(outDir + 'testConfig.yml')
except:
errDlg2 = wx.MessageDialog(
self, 'A problem occurred with preparing the pipeline test run. \n\n' \
'Please ensure you have rights access to the directories you' \
' have chosen for the CPAC working, crash, and output folders.',
'Test Configuration Error',
wx.OK | wx.ICON_ERROR)
errDlg2.ShowModal()
errDlg2.Destroy()
if (1 in c.runNuisance) or (c.Corrections != None):
strategies = sorted(build_strategies(c))
else:
strategies = None
# Run the actual pipeline building prep and see if it works or not
try:
testDlg1 = wx.MessageDialog(
self, 'Click OK to run the test. This should take only a few seconds.',
'Running Test',
wx.OK | wx.ICON_INFORMATION)
testDlg1.ShowModal()
prep_workflow(sublist[0], c, strategies, 0)
except:
testDlg1.Destroy()
errDlg1 = wx.MessageDialog(
self, 'There are issues with the current configuration which need to be' \
' resolved - please check to make sure the options you are running' \
' have the proper pre-requisites selected.',
'Pipeline Not Ready',
wx.OK | wx.ICON_ERROR)
errDlg1.ShowModal()
errDlg1.Destroy()
else:
testDlg1.Destroy()
okDlg1 = wx.MessageDialog(
self, 'The current configuration will run successfully. You can safely' \
' save and run this setup!',
'Pipeline Ready',
wx.OK | wx.ICON_INFORMATION)
okDlg1.ShowModal()
okDlg1.Destroy()
