def run(config_file, subject_list_file, p_name=None, plugin=None, plugin_args=None, tracking=True, num_subs_at_once=None): ''' ''' # Import packages import commands import os import pickle import time from CPAC.pipeline.cpac_pipeline import prep_workflow # Init variables config_file = os.path.realpath(config_file) subject_list_file = os.path.realpath(subject_list_file) # take date+time stamp for run identification purposes unique_pipeline_id = strftime("%Y%m%d%H%M%S") pipeline_start_stamp = strftime("%Y-%m-%d_%H:%M:%S") # Load in pipeline config file try: if not os.path.exists(config_file): raise IOError else: c = Configuration(yaml.load(open(config_file, 'r'))) except IOError: print "config file %s doesn't exist" % config_file raise except Exception as e: raise Exception("Error reading config file - {0}\n\nError details:" "\n{1}\n\n".format(config_file, e)) c.logDirectory = os.path.abspath(c.logDirectory) c.workingDirectory = os.path.abspath(c.workingDirectory) c.outputDirectory = os.path.abspath(c.outputDirectory) c.crashLogDirectory = os.path.abspath(c.crashLogDirectory) if num_subs_at_once: if not str(num_subs_at_once).isdigit(): raise Exception('[!] Value entered for --num_cores not a digit.') c.numParticipantsAtOnce = int(num_subs_at_once) # Do some validation validate(c) # Get the pipeline name p_name = p_name or c.pipelineName # Load in subject list try: with open(subject_list_file, 'r') as sf: sublist = yaml.load(sf) except: print "Subject list is not in proper YAML format. Please check " \ "your file" raise Exception # NOTE: strategies list is only needed in cpac_pipeline prep_workflow for # creating symlinks strategies = sorted(build_strategies(c)) # Populate subject scan map sub_scan_map = {} try: for sub in sublist: if sub['unique_id']: s = sub['subject_id'] + "_" + sub["unique_id"] else: s = sub['subject_id'] scan_ids = ['scan_anat'] if 'func' in sub: for id in sub['func']: scan_ids.append('scan_' + str(id)) if 'rest' in sub: for id in sub['rest']: scan_ids.append('scan_' + str(id)) sub_scan_map[s] = scan_ids except: print "\n\n" + "ERROR: Subject list file not in proper format - " \ "check if you loaded the correct file?" + "\n" + \ "Error name: cpac_runner_0001" + "\n\n" raise Exception pipeline_timing_info = [] pipeline_timing_info.append(unique_pipeline_id) pipeline_timing_info.append(pipeline_start_stamp) pipeline_timing_info.append(len(sublist)) if tracking: track_run(level='participant', participants=len(sublist)) # If we're running on cluster, execute job scheduler if c.runOnGrid: # Create cluster log dir cluster_files_dir = os.path.join(c.logDirectory, 'cluster_files') if not os.path.exists(cluster_files_dir): os.makedirs(cluster_files_dir) # Create strategies file strategies_file = os.path.join(cluster_files_dir, 'strategies.pkl') with open(strategies_file, 'w') as f: pickle.dump(strategies, f) # Check if its a condor job, and run that if 'condor' in c.resourceManager.lower(): run_condor_jobs(c, config_file, strategies_file, subject_list_file, p_name) # All other schedulers are supported else: run_cpac_on_cluster(config_file, subject_list_file, strategies_file, cluster_files_dir) # Run on one computer else: if not os.path.exists(c.workingDirectory): try: os.makedirs(c.workingDirectory) except: err = "\n\n[!] CPAC says: Could not create the working " \ "directory: %s\n\nMake sure you have permissions " \ "to write to this directory.\n\n" % c.workingDirectory raise Exception(err) # If it only allows one, run it linearly if c.numParticipantsAtOnce == 1: for sub in sublist: prep_workflow(sub, c, strategies, 1, pipeline_timing_info, p_name, plugin, plugin_args) return pid = open(os.path.join(c.workingDirectory, 'pid.txt'), 'w') # Init job queue job_queue = [] # Allocate processes processes = [ Process(target=prep_workflow, args=(sub, c, strategies, 1, pipeline_timing_info, p_name, plugin, plugin_args)) for sub in sublist ] # If we're allocating more processes than are subjects, run them all if len(sublist) <= c.numParticipantsAtOnce: for p in processes: p.start() print >> pid, p.pid # Otherwise manage resources to run processes incrementally else: idx = 0 while idx < len(sublist): # If the job queue is empty and we haven't started indexing if len(job_queue) == 0 and idx == 0: # Init subject process index idc = idx # Launch processes (one for each subject) for p in processes[idc:idc + c.numParticipantsAtOnce]: p.start() print >> pid, p.pid job_queue.append(p) idx += 1 # Otherwise, jobs are running - check them else: # Check every job in the queue's status for job in job_queue: # If the job is not alive if not job.is_alive(): # Find job and delete it from queue print 'found dead job ', job loc = job_queue.index(job) del job_queue[loc] # ...and start the next available process # (subject) processes[idx].start() # Append this to job queue and increment index job_queue.append(processes[idx]) idx += 1 # Add sleep so while loop isn't consuming 100% of CPU time.sleep(2) # Close PID txt file to indicate finish pid.close()
def run(config_file, subject_list_file, p_name=None, plugin=None, plugin_args=None, tracking=True, num_subs_at_once=None, debug=False): ''' ''' # Import packages import commands import os import pickle import time from CPAC.pipeline.cpac_pipeline import prep_workflow # Init variables config_file = os.path.realpath(config_file) subject_list_file = os.path.realpath(subject_list_file) # take date+time stamp for run identification purposes unique_pipeline_id = strftime("%Y%m%d%H%M%S") pipeline_start_stamp = strftime("%Y-%m-%d_%H:%M:%S") # Load in pipeline config file try: if not os.path.exists(config_file): raise IOError else: c = Configuration(yaml.load(open(config_file, 'r'))) except IOError: print "config file %s doesn't exist" % config_file raise except Exception as e: raise Exception("Error reading config file - {0}\n\nError details:" "\n{1}\n\n".format(config_file, e)) c.logDirectory = os.path.abspath(c.logDirectory) c.workingDirectory = os.path.abspath(c.workingDirectory) if 's3://' not in c.outputDirectory: c.outputDirectory = os.path.abspath(c.outputDirectory) c.crashLogDirectory = os.path.abspath(c.crashLogDirectory) if debug: c.write_debugging_outputs = "[1]" if num_subs_at_once: if not str(num_subs_at_once).isdigit(): raise Exception('[!] Value entered for --num_cores not a digit.') c.numParticipantsAtOnce = int(num_subs_at_once) # Do some validation validate(c) # Get the pipeline name p_name = p_name or c.pipelineName # Load in subject list try: with open(subject_list_file, 'r') as sf: sublist = yaml.load(sf) except: print "Subject list is not in proper YAML format. Please check " \ "your file" raise Exception # Populate subject scan map sub_scan_map = {} try: for sub in sublist: if sub['unique_id']: s = sub['subject_id'] + "_" + sub["unique_id"] else: s = sub['subject_id'] scan_ids = ['scan_anat'] if 'func' in sub: for id in sub['func']: scan_ids.append('scan_'+ str(id)) if 'rest' in sub: for id in sub['rest']: scan_ids.append('scan_'+ str(id)) sub_scan_map[s] = scan_ids except: print "\n\n" + "ERROR: Subject list file not in proper format - " \ "check if you loaded the correct file?" + "\n" + \ "Error name: cpac_runner_0001" + "\n\n" raise Exception pipeline_timing_info = [] pipeline_timing_info.append(unique_pipeline_id) pipeline_timing_info.append(pipeline_start_stamp) pipeline_timing_info.append(len(sublist)) if tracking: track_run(level='participant', participants=len(sublist)) # If we're running on cluster, execute job scheduler if c.runOnGrid: # Create cluster log dir cluster_files_dir = os.path.join(c.logDirectory, 'cluster_files') if not os.path.exists(cluster_files_dir): os.makedirs(cluster_files_dir) # Check if its a condor job, and run that if 'condor' in c.resourceManager.lower(): run_condor_jobs(c, config_file, subject_list_file, p_name) # All other schedulers are supported else: run_cpac_on_cluster(config_file, subject_list_file, cluster_files_dir) # Run on one computer else: if not os.path.exists(c.workingDirectory): try: os.makedirs(c.workingDirectory) except: err = "\n\n[!] CPAC says: Could not create the working " \ "directory: %s\n\nMake sure you have permissions " \ "to write to this directory.\n\n" % c.workingDirectory raise Exception(err) # If it only allows one, run it linearly if c.numParticipantsAtOnce == 1: for sub in sublist: prep_workflow(sub, c, True, pipeline_timing_info, p_name, plugin, plugin_args) return pid = open(os.path.join(c.workingDirectory, 'pid.txt'), 'w') # Init job queue job_queue = [] # Allocate processes processes = [Process(target=prep_workflow, args=(sub, c, True, pipeline_timing_info, p_name, plugin, plugin_args)) for sub in sublist] # If we're allocating more processes than are subjects, run them all if len(sublist) <= c.numParticipantsAtOnce: for p in processes: p.start() print >>pid, p.pid # Otherwise manage resources to run processes incrementally else: idx = 0 while idx < len(sublist): # If the job queue is empty and we haven't started indexing if len(job_queue) == 0 and idx == 0: # Init subject process index idc = idx # Launch processes (one for each subject) for p in processes[idc: idc+c.numParticipantsAtOnce]: p.start() print >>pid, p.pid job_queue.append(p) idx += 1 # Otherwise, jobs are running - check them else: # Check every job in the queue's status for job in job_queue: # If the job is not alive if not job.is_alive(): # Find job and delete it from queue print 'found dead job ', job loc = job_queue.index(job) del job_queue[loc] # ...and start the next available process # (subject) processes[idx].start() # Append this to job queue and increment index job_queue.append(processes[idx]) idx += 1 # Add sleep so while loop isn't consuming 100% of CPU time.sleep(2) # Close PID txt file to indicate finish pid.close()
def run(subject_list_file, config_file=None, p_name=None, plugin=None, plugin_args=None, tracking=True, num_subs_at_once=None, debug=False, test_config=False): # Import packages import commands import os import pickle import time from CPAC.pipeline.cpac_pipeline import prep_workflow print('Run called with config file {0}'.format(config_file)) if not config_file: import pkg_resources as p config_file = \ p.resource_filename("CPAC", os.path.join("resources", "configs", "pipeline_config_template.yml")) # Init variables sublist = None config_file = os.path.realpath(config_file) if '.yaml' in subject_list_file or '.yml' in subject_list_file: subject_list_file = os.path.realpath(subject_list_file) else: from CPAC.utils.bids_utils import collect_bids_files_configs, \ bids_gen_cpac_sublist (file_paths, config) = collect_bids_files_configs(subject_list_file, None) sublist = bids_gen_cpac_sublist(subject_list_file, file_paths, config, None) if not sublist: import sys print("Did not find data in {0}".format(subject_list_file)) sys.exit(1) # take date+time stamp for run identification purposes unique_pipeline_id = strftime("%Y%m%d%H%M%S") pipeline_start_stamp = strftime("%Y-%m-%d_%H:%M:%S") # Load in pipeline config file try: if not os.path.exists(config_file): raise IOError else: c = Configuration(yaml.load(open(config_file, 'r'))) except IOError: print "config file %s doesn't exist" % config_file raise except yaml.parser.ParserError as e: error_detail = "\"%s\" at line %d" % ( e.problem, e.problem_mark.line ) raise Exception( "Error parsing config file: {0}\n\n" "Error details:\n" " {1}" "\n\n".format(config_file, error_detail) ) except Exception as e: raise Exception( "Error parsing config file: {0}\n\n" "Error details:\n" " {1}" "\n\n".format(config_file, e) ) c.logDirectory = os.path.abspath(c.logDirectory) c.workingDirectory = os.path.abspath(c.workingDirectory) if 's3://' not in c.outputDirectory: c.outputDirectory = os.path.abspath(c.outputDirectory) c.crashLogDirectory = os.path.abspath(c.crashLogDirectory) if debug: c.write_debugging_outputs = "[1]" if num_subs_at_once: if not str(num_subs_at_once).isdigit(): raise Exception('[!] Value entered for --num_cores not a digit.') c.numParticipantsAtOnce = int(num_subs_at_once) # Do some validation if not c.workingDirectory: raise Exception('Working directory not specified') if len(c.workingDirectory) > 70: warnings.warn("We recommend that the working directory full path " "should have less then 70 characters. " "Long paths might not work in your operational system.") warnings.warn("Current working directory: %s" % c.workingDirectory) # Get the pipeline name p_name = p_name or c.pipelineName # Load in subject list try: if not sublist: with open(subject_list_file, 'r') as sf: sublist = yaml.load(sf) except: print "Subject list is not in proper YAML format. Please check " \ "your file" raise Exception # Populate subject scan map sub_scan_map = {} try: for sub in sublist: if sub['unique_id']: s = sub['subject_id'] + "_" + sub["unique_id"] else: s = sub['subject_id'] scan_ids = ['scan_anat'] if 'func' in sub: for id in sub['func']: scan_ids.append('scan_'+ str(id)) if 'rest' in sub: for id in sub['rest']: scan_ids.append('scan_'+ str(id)) sub_scan_map[s] = scan_ids except: print "\n\n" + "ERROR: Subject list file not in proper format - " \ "check if you loaded the correct file?" + "\n" + \ "Error name: cpac_runner_0001" + "\n\n" raise Exception pipeline_timing_info = [] pipeline_timing_info.append(unique_pipeline_id) pipeline_timing_info.append(pipeline_start_stamp) pipeline_timing_info.append(len(sublist)) if tracking: try: track_run(level='participant', participants=len(sublist)) except: pass # If we're running on cluster, execute job scheduler if c.runOnGrid: # Create cluster log dir cluster_files_dir = os.path.join(c.logDirectory, 'cluster_files') if not os.path.exists(cluster_files_dir): os.makedirs(cluster_files_dir) # Check if its a condor job, and run that if 'condor' in c.resourceManager.lower(): run_condor_jobs(c, config_file, subject_list_file, p_name) # All other schedulers are supported else: run_cpac_on_cluster(config_file, subject_list_file, cluster_files_dir) # Run on one computer else: if not os.path.exists(c.workingDirectory): try: os.makedirs(c.workingDirectory) except: err = "\n\n[!] CPAC says: Could not create the working " \ "directory: %s\n\nMake sure you have permissions " \ "to write to this directory.\n\n" % c.workingDirectory raise Exception(err) # If it only allows one, run it linearly if c.numParticipantsAtOnce == 1: for sub in sublist: prep_workflow(sub, c, True, pipeline_timing_info, p_name, plugin, plugin_args, test_config) return pid = open(os.path.join(c.workingDirectory, 'pid.txt'), 'w') # Init job queue job_queue = [] # Allocate processes processes = [ Process(target=prep_workflow, args=(sub, c, True, pipeline_timing_info, p_name, plugin, plugin_args, test_config)) for sub in sublist ] # If we're allocating more processes than are subjects, run them all if len(sublist) <= c.numParticipantsAtOnce: for p in processes: p.start() print >>pid, p.pid # Otherwise manage resources to run processes incrementally else: idx = 0 while idx < len(sublist): # If the job queue is empty and we haven't started indexing if len(job_queue) == 0 and idx == 0: # Init subject process index idc = idx # Launch processes (one for each subject) for p in processes[idc: idc+c.numParticipantsAtOnce]: p.start() print >>pid, p.pid job_queue.append(p) idx += 1 # Otherwise, jobs are running - check them else: # Check every job in the queue's status for job in job_queue: # If the job is not alive if not job.is_alive(): # Find job and delete it from queue print 'found dead job ', job loc = job_queue.index(job) del job_queue[loc] # ...and start the next available process # (subject) processes[idx].start() # Append this to job queue and increment index job_queue.append(processes[idx]) idx += 1 # Add sleep so while loop isn't consuming 100% of CPU time.sleep(2) # Close PID txt file to indicate finish pid.close()
def testConfig(self, event): ''' This function runs when the user clicks the "Test Configuration" button in the pipeline configuration window. It prompts the user for a sample subject list (i.e. one that they will be using with the config they are building). Then it builds the pipeline but does not run it. It then reports whether or not the config will run or not depending on if the pipeline gets built successfully. ''' import os import yaml from CPAC.utils import Configuration from CPAC.pipeline.cpac_pipeline import prep_workflow from CPAC.pipeline.cpac_runner import build_strategies def display(win, msg, changeBg=True): wx.MessageBox(msg, "Error") if changeBg: win.SetBackgroundColour("pink") win.SetFocus() win.Refresh() # Collect a sample subject list and parse it in testDlg0 = wx.MessageDialog( self, 'This tool will run a quick check on the current pipeline configuration.' \ ' Click OK to provide a subject list you will be using with this setup.', 'Subject List', wx.OK | wx.ICON_INFORMATION) testDlg0.ShowModal() testDlg0.Destroy() dlg = wx.FileDialog( self, message="Choose the CPAC Subject list file", defaultDir=os.getcwd(), defaultFile="CPAC_subject_list.yml", wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml", style=wx.OPEN | wx.CHANGE_DIR) if dlg.ShowModal() == wx.ID_OK: subListPath = dlg.GetPath() # Load and test the subject list sublist = yaml.load(open(os.path.realpath(subListPath), 'r')) sub_flg = self.test_sublist(sublist) if not sub_flg: raise Exception # Following code reads in the parameters and selections from the # pipeline configuration window and populate the config_list config_list = [] wf_counter = [] for page in self.nb.get_page_list(): switch = page.page.get_switch() ctrl_list = page.page.get_ctrl_list() validate = False if switch: switch_val = str(switch.get_selection()).lower() if switch_val == 'on' or switch_val == 'true' or switch_val == '1': validate = True wf_counter.append(page.get_counter()) for ctrl in ctrl_list: # option_name will be the selection name as it is written # as the dictionary key of the config.yml dictionary option_name = ctrl.get_name() #validating if (switch == None or validate) and ctrl.get_validation() \ and (option_name != 'derivativeList') and (option_name != 'modelConfigs'): win = ctrl.get_ctrl() if isinstance(ctrl.get_selection(), list): value = ctrl.get_selection() if not value: display( win, "%s field is empty or the items are not checked!" % ctrl.get_name(), False) return else: value = str(ctrl.get_selection()) if len(value) == 0: display(win, "%s field is empty!" % ctrl.get_name()) return if '/' in value and '$' not in value and not isinstance(value, list): if not os.path.exists(ctrl.get_selection()) and value != 'On/Off': display( win, "%s field contains incorrect path. Please update the path!" % ctrl.get_name()) return config_list.append(ctrl) # Get the user's CPAC output directory for use in this script for config in config_list: if config.get_name() == 'outputDirectory': outDir = config.get_selection() # Write out a pipeline_config file, read it in and then delete it # (Will revise the data structure of the config files later so this # can just pass the data structure instead of doing it this way) try: self.write(outDir + 'testConfig.yml', config_list) c = Configuration(yaml.load(open(os.path.realpath(outDir + 'testConfig.yml'), 'r'))) os.remove(outDir + 'testConfig.yml') except: errDlg2 = wx.MessageDialog( self, 'A problem occurred with preparing the pipeline test run. \n\n' \ 'Please ensure you have rights access to the directories you' \ ' have chosen for the CPAC working, crash, and output folders.', 'Test Configuration Error', wx.OK | wx.ICON_ERROR) errDlg2.ShowModal() errDlg2.Destroy() if (1 in c.runNuisance) or (c.Corrections != None): strategies = sorted(build_strategies(c)) else: strategies = None # Run the actual pipeline building prep and see if it works or not testDlg1 = wx.MessageDialog( self, 'Click OK to run the test. This should take only a few seconds.', 'Running Test', wx.OK | wx.ICON_INFORMATION) testDlg1.ShowModal() # Check file paths first # Just getting proper names of config file parameters try: params_file = open(p.resource_filename('CPAC', 'GUI/resources/config_parameters.txt'), "r") except: print "Error: Could not open configuration parameter file.", "\n" raise Exception paramInfo = params_file.read().split('\n') paramList = [] for param in paramInfo: if param != '': paramList.append(param.split(',')) # function for file path checking def testFile(filepath, paramName, switch): try: if (1 in switch) and (filepath != None): fileTest = open(filepath) fileTest.close() except: testDlg1.Destroy() for param in paramList: if param[0] == paramName: paramTitle = param[1] paramGroup = param[2] break errDlgFileTest = wx.MessageDialog( self, 'Error reading file - either it does not exist or you' \ ' do not have read access. \n\n' \ 'Parameter: %s \n' \ 'In tab: %s \n\n' \ 'Path: %s' % (paramTitle, paramGroup, filepath), 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlgFileTest.ShowModal() errDlgFileTest.Destroy() testFile(c.template_brain_only_for_anat,'template_brain_only_for_anat',c.runRegistrationPreprocessing) testFile(c.template_skull_for_anat,'template_skull_for_anat',c.runRegistrationPreprocessing) testFile(c.PRIORS_WHITE,'PRIORS_WHITE',c.runSegmentationPreprocessing) testFile(c.PRIORS_GRAY,'PRIORS_GRAY',c.runSegmentationPreprocessing) testFile(c.PRIORS_CSF,'PRIORS_CSF',c.runSegmentationPreprocessing) testFile(c.template_brain_only_for_func,'template_brain_only_for_func',c.runRegisterFuncToMNI) testFile(c.template_skull_for_func,'template_skull_for_func',c.runRegisterFuncToMNI) testFile(c.identityMatrix,'identityMatrix',c.runRegisterFuncToMNI) testFile(c.boundaryBasedRegistrationSchedule,'boundaryBasedRegistrationSchedule',c.runRegisterFuncToAnat) testFile(c.lateral_ventricles_mask,'lateral_ventricles_mask',c.runNuisance) testFile(c.seedSpecificationFile,'seedSpecificationFile',[1]) testFile(c.roiSpecificationFile,'roiSpecificationFile',c.runROITimeseries) testFile(c.roiSpecificationFileForSCA,'roiSpecificationFileForSCA',c.runROITimeseries) testFile(c.maskSpecificationFile,'maskSpecificationFile',c.runVoxelTimeseries) testFile(c.maskSpecificationFileForSCA,'maskSpecificationFileForSCA',c.runVoxelTimeseries) testFile(c.spatialPatternMaps,'spatialPatternMaps',c.runSpatialRegression) testFile(c.template_symmetric_brain_only,'template_symmetric_brain_only',c.runVMHC) testFile(c.template_symmetric_skull,'template_symmetric_skull',c.runVMHC) testFile(c.dilated_symmetric_brain_mask,'dilated_symmetric_brain_mask',c.runVMHC) testFile(c.configFileTwomm,'configFileTwomm',c.runVMHC) testFile(c.templateSpecificationFile,'templateSpecificationFile',c.runNetworkCentrality) testFile(c.bascAffinityThresholdFile,'bascAffinityThresholdFile',c.runBASC) testFile(c.cwasROIFile,'cwasROIFile',c.runCWAS) testFile(c.cwasRegressorFile,'cwasRegressorFile',c.runCWAS) try: # Run the pipeline building prep_workflow(sublist[0], c, strategies, 0) except Exception as xxx: print xxx print "an exception occured" testDlg1.Destroy() errDlg1 = wx.MessageDialog( self, 'There are issues with the current configuration ' \ 'which need to be resolved - please check to make ' \ 'sure the options you are running have the proper ' \ 'pre-requisites selected.\n\nIssue Info:\n%s' % xxx, 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlg1.ShowModal() errDlg1.Destroy() else: testDlg1.Destroy() okDlg1 = wx.MessageDialog( self, 'The current configuration will run successfully. You can safely' \ ' save and run this setup!', 'Pipeline Ready', wx.OK | wx.ICON_INFORMATION) okDlg1.ShowModal() okDlg1.Destroy()
def testConfig(self, event): ''' This function runs when the user clicks the "Test Configuration" button in the pipeline configuration window. It prompts the user for a sample subject list (i.e. one that they will be using with the config they are building). Then it builds the pipeline but does not run it. It then reports whether or not the config will run or not depending on if the pipeline gets built successfully. ''' # Import packages import os import yaml from CPAC.utils import Configuration from CPAC.pipeline.cpac_pipeline import prep_workflow from CPAC.pipeline.cpac_runner import build_strategies def display(win, msg, changeBg=True): wx.MessageBox(msg, "Error") if changeBg: win.SetBackgroundColour("pink") win.SetFocus() win.Refresh() # Collect a sample subject list and parse it in testDlg0 = wx.MessageDialog( self, 'This tool will run a quick check on the current pipeline '\ 'configuration. Click OK to provide a subject list you ' \ 'will be using with this setup.', 'Subject List', wx.OK | wx.ICON_INFORMATION) testDlg0.ShowModal() testDlg0.Destroy() dlg = wx.FileDialog(self, message="Choose the CPAC Subject list file", defaultDir=os.getcwd(), defaultFile="CPAC_subject_list.yml", wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml", style=wx.OPEN | wx.CHANGE_DIR) if dlg.ShowModal() == wx.ID_OK: subListPath = dlg.GetPath() # Load and test the subject list print 'Checking subject list: %s...' % subListPath sublist = yaml.load(open(os.path.realpath(subListPath), 'r')) sub_flg = self.test_sublist(sublist) if not sub_flg: raise Exception print 'Subject list looks good!' # Following code reads in the parameters and selections from the # pipeline configuration window and populate the config_list config_list = [] wf_counter = [] for page in self.nb.get_page_list(): switch = page.page.get_switch() ctrl_list = page.page.get_ctrl_list() validate = False if switch: switch_val = str(switch.get_selection()).lower() if switch_val == 'on' or switch_val == 'true' or \ switch_val == '1': validate = True wf_counter.append(page.get_counter()) for ctrl in ctrl_list: # option_name will be the selection name as it is written # as the dictionary key of the config.yml dictionary option_name = ctrl.get_name() #validating if (switch == None or validate) and ctrl.get_validation() \ and (option_name != 'derivativeList') and \ (option_name != 'modelConfigs'): win = ctrl.get_ctrl() if isinstance(ctrl.get_selection(), list): value = ctrl.get_selection() if not value: display( win, "%s field is empty or the items are " \ "not checked!" % ctrl.get_name(), False) return elif (option_name == "tsa_roi_paths") or \ (option_name == "sca_roi_paths"): # fires if the control is the checkbox grid for # multiple paths assigned to multiple options # (i.e. timeseries analysis) config_list.append(ctrl) continue else: value = str(ctrl.get_selection()) if len(value) == 0: display(win, "%s field is empty!" % ctrl.get_name()) return if '/' in value and '$' not in value and not \ isinstance(value, list): if not os.path.exists(ctrl.get_selection()) and \ value != 'On/Off': display( win, "%s field contains incorrect path. " \ "Please update the path!" % ctrl.get_name()) return config_list.append(ctrl) # Write out a pipeline_config file, read it in and then delete it # (Will revise the data structure of the config files later so this # can just pass the data structure instead of doing it this way) try: test_cfg_yml = '/tmp/test_config.yml' self.write(test_cfg_yml, config_list) c = Configuration( yaml.load(open(os.path.realpath(test_cfg_yml), 'r'))) os.remove(test_cfg_yml) except: errDlg2 = wx.MessageDialog( self, 'A problem occurred with preparing the pipeline test run. \n\n' \ 'Please ensure you have rights access to the directories you' \ ' have chosen for the CPAC working, crash, and output folders.', 'Test Configuration Error', wx.OK | wx.ICON_ERROR) errDlg2.ShowModal() errDlg2.Destroy() if (1 in c.runNuisance) or (c.Regressors != None): strategies = sorted(build_strategies(c)) else: strategies = None # Run the actual pipeline building prep and see if it works or not testDlg1 = wx.MessageDialog( self, 'Click OK to run the test. This should take only a few seconds.', 'Running Test', wx.OK | wx.ICON_INFORMATION) testDlg1.ShowModal() # Check file paths first # Just getting proper names of config file parameters try: params_file = open( p.resource_filename('CPAC', 'GUI/resources/config_parameters.txt'), "r") except: print "Error: Could not open configuration parameter file.", "\n" raise Exception paramInfo = params_file.read().split('\n') paramList = [] for param in paramInfo: if param != '': paramList.append(param.split(',')) # function for file path checking def testFile(filepath, paramName, switch): try: if (1 in switch) and (filepath != None): fileTest = open(filepath) fileTest.close() except: testDlg1.Destroy() for param in paramList: if param[0] == paramName: paramTitle = param[1] paramGroup = param[2] break errDlgFileTest = wx.MessageDialog( self, 'Error reading file - either it does not exist or '\ 'you do not have read access. \n\n' \ 'Parameter: %s \n' \ 'In tab: %s \n\n' \ 'Path: %s' % (paramTitle, paramGroup, filepath), 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlgFileTest.ShowModal() errDlgFileTest.Destroy() # Check S3 output bucket access if writing to S3 output_dir = c.outputDirectory s3_str = 's3://' if output_dir.lower().startswith(s3_str): output_dir_sp = output_dir.split('/') output_dir_sp[0] = output_dir_sp[0].lower() output_dir = '/'.join(output_dir_sp) if type(output_dir) is str and output_dir.lower().startswith(s3_str): from indi_aws import fetch_creds creds_path = c.awsOutputBucketCredentials bucket_name = output_dir.split(s3_str)[1].split('/')[0] try: bucket = fetch_creds.return_bucket(creds_path, bucket_name) print 'Connection with output bucket "%s" successful!' % bucket_name except Exception as exc: err_msg = 'Unable to access output S3 bucket: "%s" with '\ 'credentials in: "%s". Check bucket name '\ 'and credentials file and try again'\ % (bucket_name, creds_path) testDlg1.Destroy() errDlg1 = wx.MessageDialog(self, err_msg, 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlg1.ShowModal() errDlg1.Destroy() return testFile(c.template_brain_only_for_anat, \ 'template_brain_only_for_anat',[1]) testFile(c.template_skull_for_anat, 'template_skull_for_anat', [1]) testFile(c.PRIORS_WHITE, 'PRIORS_WHITE', c.runSegmentationPreprocessing) testFile(c.PRIORS_GRAY, 'PRIORS_GRAY', c.runSegmentationPreprocessing) testFile(c.PRIORS_CSF, 'PRIORS_CSF', c.runSegmentationPreprocessing) testFile(c.template_brain_only_for_func, \ 'template_brain_only_for_func',c.runRegisterFuncToMNI) testFile(c.template_skull_for_func,'template_skull_for_func', \ c.runRegisterFuncToMNI) testFile(c.identityMatrix, 'identityMatrix', c.runRegisterFuncToMNI) testFile(c.boundaryBasedRegistrationSchedule, \ 'boundaryBasedRegistrationSchedule', \ c.runRegisterFuncToAnat) testFile(c.lateral_ventricles_mask,'lateral_ventricles_mask', \ c.runNuisance) testFile(c.template_symmetric_brain_only, \ 'template_symmetric_brain_only',c.runVMHC) testFile(c.template_symmetric_skull,'template_symmetric_skull', \ c.runVMHC) testFile(c.dilated_symmetric_brain_mask, \ 'dilated_symmetric_brain_mask',c.runVMHC) testFile(c.configFileTwomm, 'configFileTwomm', c.runVMHC) testFile(c.templateSpecificationFile,'templateSpecificationFile', \ c.runNetworkCentrality) if c.tsa_roi_paths and type(c.tsa_roi_paths[0]) == dict: for roi_path in c.tsa_roi_paths[0].keys(): testFile(roi_path, "tsa_roi_paths", c.runROITimeseries) if c.sca_roi_paths and type(c.sca_roi_paths[0]) == dict: for roi_path in c.sca_roi_paths[0].keys(): testFile(roi_path, "sca_roi_paths", c.runSCA) try: # Run the pipeline building prep_workflow(sublist[0], c, strategies, 0) except Exception as xxx: print xxx print "an exception occurred" testDlg1.Destroy() errDlg1 = wx.MessageDialog( self, 'There are issues with the current configuration ' \ 'which need to be resolved - please check to make ' \ 'sure the options you are running have the proper ' \ 'pre-requisites selected.\n\nIssue Info:\n%s' \ % str(xxx), 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlg1.ShowModal() errDlg1.Destroy() else: testDlg1.Destroy() okDlg1 = wx.MessageDialog( self, 'The current configuration will run successfully. You '\ 'can safely save and run this setup!', 'Pipeline Ready', wx.OK | wx.ICON_INFORMATION) okDlg1.ShowModal() okDlg1.Destroy()
def testConfig(self, event): ''' This function runs when the user clicks the "Test Configuration" button in the pipeline configuration window. It prompts the user for a sample subject list (i.e. one that they will be using with the config they are building). Then it builds the pipeline but does not run it. It then reports whether or not the config will run or not depending on if the pipeline gets built successfully. ''' # Import packages import os import yaml from CPAC.utils import Configuration from CPAC.pipeline.cpac_pipeline import prep_workflow from CPAC.pipeline.cpac_runner import build_strategies def display(win, msg, changeBg=True): wx.MessageBox(msg, "Error") if changeBg: win.SetBackgroundColour("pink") win.SetFocus() win.Refresh() # Collect a sample subject list and parse it in testDlg0 = wx.MessageDialog( self, 'This tool will run a quick check on the current pipeline ' 'configuration. Click OK to provide a subject list you ' 'will be using with this setup.', 'Subject List', wx.OK | wx.ICON_INFORMATION) testDlg0.ShowModal() testDlg0.Destroy() dlg = wx.FileDialog( self, message="Choose the CPAC Subject list file", defaultDir=os.getcwd(), defaultFile="CPAC_subject_list.yml", wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml", style=wx.OPEN | wx.CHANGE_DIR) if dlg.ShowModal() == wx.ID_OK: subListPath = dlg.GetPath() # Load and test the subject list print 'Checking subject list: %s...' % subListPath sublist = yaml.load(open(os.path.realpath(subListPath), 'r')) sub_flg = self.test_sublist(sublist) if not sub_flg: raise Exception print 'Subject list looks good!' # Following code reads in the parameters and selections from the # pipeline configuration window and populate the config_list config_list = [] wf_counter = [] for page in self.nb.get_page_list(): switch = page.page.get_switch() ctrl_list = page.page.get_ctrl_list() validate = False if switch: switch_val = str(switch.get_selection()).lower() if switch_val == 'on' or switch_val == 'true' or \ switch_val == '1': validate = True wf_counter.append(page.get_counter()) for ctrl in ctrl_list: # option_name will be the selection name as it is written # as the dictionary key of the config.yml dictionary option_name = ctrl.get_name() #validating if (switch == None or validate) and ctrl.get_validation() \ and (option_name != 'derivativeList') and \ (option_name != 'modelConfigs'): win = ctrl.get_ctrl() if isinstance(ctrl.get_selection(), list): value = ctrl.get_selection() if not value: display( win, "%s field is empty or the items are " \ "not checked!" % ctrl.get_name(), False) return elif (option_name == "tsa_roi_paths") or \ (option_name == "sca_roi_paths"): # fires if the control is the checkbox grid for # multiple paths assigned to multiple options # (i.e. timeseries analysis) config_list.append(ctrl) continue else: value = str(ctrl.get_selection()) if len(value) == 0: display(win, "%s field is empty!" % ctrl.get_name()) return if '/' in value and '$' not in value and not \ isinstance(value, list): if not os.path.exists(ctrl.get_selection()) and \ value != 'On/Off': display( win, "%s field contains incorrect path. " \ "Please update the path!" % ctrl.get_name()) return config_list.append(ctrl) # Write out a pipeline_config file, read it in and then delete it # (Will revise the data structure of the config files later so this # can just pass the data structure instead of doing it this way) try: test_cfg_yml = '/tmp/test_config.yml' self.write(test_cfg_yml, config_list) c = Configuration(yaml.load(open(os.path.realpath(test_cfg_yml), 'r'))) os.remove(test_cfg_yml) except: errDlg2 = wx.MessageDialog( self, 'A problem occurred with preparing the pipeline test run. \n\n' \ 'Please ensure you have rights access to the directories you' \ ' have chosen for the CPAC working, crash, and output folders.', 'Test Configuration Error', wx.OK | wx.ICON_ERROR) errDlg2.ShowModal() errDlg2.Destroy() if (1 in c.runNuisance) or (c.Regressors != None): strategies = sorted(build_strategies(c)) else: strategies = None # Run the actual pipeline building prep and see if it works or not testDlg1 = wx.MessageDialog( self, 'Click OK to run the test. This should take only a few seconds.', 'Running Test', wx.OK | wx.ICON_INFORMATION) testDlg1.ShowModal() # Check file paths first # Just getting proper names of config file parameters try: params_file = open(p.resource_filename('CPAC', 'GUI/resources/config_parameters.txt'), "r") except: print "Error: Could not open configuration parameter file.", "\n" raise Exception paramInfo = params_file.read().split('\n') paramList = [] for param in paramInfo: if param != '': paramList.append(param.split(',')) # function for file path checking def testFile(filepath, paramName, switch): try: if (1 in switch) and (filepath != None): fileTest = open(filepath) fileTest.close() except: testDlg1.Destroy() for param in paramList: if param[0] == paramName: paramTitle = param[1] paramGroup = param[2] break errDlgFileTest = wx.MessageDialog( self, 'Error reading file - either it does not exist or '\ 'you do not have read access. \n\n' \ 'Parameter: %s \n' \ 'In tab: %s \n\n' \ 'Path: %s' % (paramTitle, paramGroup, filepath), 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlgFileTest.ShowModal() errDlgFileTest.Destroy() # Check S3 output bucket access if writing to S3 output_dir = c.outputDirectory s3_str = 's3://' if output_dir.lower().startswith(s3_str): output_dir_sp = output_dir.split('/') output_dir_sp[0] = output_dir_sp[0].lower() output_dir = '/'.join(output_dir_sp) if type(output_dir) is str and output_dir.lower().startswith(s3_str): from indi_aws import fetch_creds creds_path = c.awsOutputBucketCredentials bucket_name = output_dir.split(s3_str)[1].split('/')[0] try: bucket = fetch_creds.return_bucket(creds_path, bucket_name) print 'Connection with output bucket "%s" successful!' % bucket_name except Exception as exc: err_msg = 'Unable to access output S3 bucket: "%s" with '\ 'credentials in: "%s". Check bucket name '\ 'and credentials file and try again'\ % (bucket_name, creds_path) testDlg1.Destroy() errDlg1 = wx.MessageDialog(self, err_msg, 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlg1.ShowModal() errDlg1.Destroy() return testFile(c.template_brain_only_for_anat, \ 'template_brain_only_for_anat',[1]) testFile(c.template_skull_for_anat,'template_skull_for_anat',[1]) testFile(c.PRIORS_WHITE,'PRIORS_WHITE',c.runSegmentationPreprocessing) testFile(c.PRIORS_GRAY,'PRIORS_GRAY',c.runSegmentationPreprocessing) testFile(c.PRIORS_CSF,'PRIORS_CSF',c.runSegmentationPreprocessing) testFile(c.template_brain_only_for_func, \ 'template_brain_only_for_func',c.runRegisterFuncToMNI) testFile(c.template_skull_for_func,'template_skull_for_func', \ c.runRegisterFuncToMNI) testFile(c.identityMatrix,'identityMatrix',c.runRegisterFuncToMNI) testFile(c.boundaryBasedRegistrationSchedule, \ 'boundaryBasedRegistrationSchedule', \ c.runRegisterFuncToAnat) testFile(c.lateral_ventricles_mask,'lateral_ventricles_mask', \ c.runNuisance) testFile(c.template_symmetric_brain_only, \ 'template_symmetric_brain_only',c.runVMHC) testFile(c.template_symmetric_skull,'template_symmetric_skull', \ c.runVMHC) testFile(c.dilated_symmetric_brain_mask, \ 'dilated_symmetric_brain_mask',c.runVMHC) testFile(c.configFileTwomm,'configFileTwomm',c.runVMHC) testFile(c.templateSpecificationFile,'templateSpecificationFile', \ c.runNetworkCentrality) if c.tsa_roi_paths and type(c.tsa_roi_paths[0]) == dict: for roi_path in c.tsa_roi_paths[0].keys(): testFile(roi_path, "tsa_roi_paths", c.runROITimeseries) if c.sca_roi_paths and type(c.sca_roi_paths[0]) == dict: for roi_path in c.sca_roi_paths[0].keys(): testFile(roi_path, "sca_roi_paths", c.runSCA) try: # Run the pipeline building prep_workflow(sublist[0], c, strategies, 0) except Exception as xxx: print xxx print "an exception occurred" testDlg1.Destroy() errDlg1 = wx.MessageDialog( self, 'There are issues with the current configuration ' \ 'which need to be resolved - please check to make ' \ 'sure the options you are running have the proper ' \ 'pre-requisites selected.\n\nIssue Info:\n%s' \ % str(xxx), 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlg1.ShowModal() errDlg1.Destroy() else: testDlg1.Destroy() okDlg1 = wx.MessageDialog( self, 'The current configuration will run successfully. You '\ 'can safely save and run this setup!', 'Pipeline Ready', wx.OK | wx.ICON_INFORMATION) okDlg1.ShowModal() okDlg1.Destroy()
def testConfig(self, event): ''' This function runs when the user clicks the "Test Configuration" button in the pipeline configuration window. It prompts the user for a sample subject list (i.e. one that they will be using with the config they are building). Then it builds the pipeline but does not run it. It then reports whether or not the config will run or not depending on if the pipeline gets built successfully. ''' import os import yaml from CPAC.utils import Configuration from CPAC.pipeline.cpac_pipeline import prep_workflow from CPAC.pipeline.cpac_runner import build_strategies def display(win, msg, changeBg=True): wx.MessageBox(msg, "Error") if changeBg: win.SetBackgroundColour("pink") win.SetFocus() win.Refresh() # Collect a sample subject list and parse it in testDlg0 = wx.MessageDialog( self, 'This tool will run a quick check on the current pipeline configuration.' \ ' Click OK to provide a subject list you will be using with this setup.', 'Subject List', wx.OK | wx.ICON_INFORMATION) testDlg0.ShowModal() testDlg0.Destroy() dlg = wx.FileDialog(self, message="Choose the CPAC Subject list file", defaultDir=os.getcwd(), defaultFile="CPAC_subject_list.yml", wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml", style=wx.OPEN | wx.CHANGE_DIR) if dlg.ShowModal() == wx.ID_OK: subListPath = dlg.GetPath() sublist = yaml.load(open(os.path.realpath(subListPath), 'r')) # Check to ensure the user is providing an actual subject # list and not some other kind of file try: subInfo = sublist[0] except: errDlg4 = wx.MessageDialog( self, 'ERROR: Subject list file not in proper format - check if you' \ ' loaded the correct file? \n\n' \ 'Error name: config_window_0001', 'Subject List Error', wx.OK | wx.ICON_ERROR) errDlg4.ShowModal() errDlg4.Destroy() raise Exception # Another check to ensure the actual subject list was generated # properly and that it will work if 'subject_id' not in subInfo: errDlg3 = wx.MessageDialog( self, 'ERROR: Subject list file not in proper format - check if you' \ ' loaded the correct file? \n\n' \ 'Error name: config_window_0002', 'Subject List Error', wx.OK | wx.ICON_ERROR) errDlg3.ShowModal() errDlg3.Destroy() raise Exception # Following code reads in the parameters and selections from the # pipeline configuration window and populate the config_list config_list = [] wf_counter = [] #print "self.nb.get_page_list()", self.nb.get_page_list() for page in self.nb.get_page_list(): #print "page ----> ", page switch = page.page.get_switch() #print "switch ---->", switch ctrl_list = page.page.get_ctrl_list() validate = False if switch: switch_val = str(switch.get_selection()).lower() #print "switch_val ---->", switch_val if switch_val == 'on' or switch_val == 'true' or switch_val == '1': validate = True wf_counter.append(page.get_counter()) for ctrl in ctrl_list: #validating if (switch == None or validate) and ctrl.get_validation(): win = ctrl.get_ctrl() #print "validating ctrl-->", ctrl.get_name() #print "ctrl.get_selection()", ctrl.get_selection() #print "type(ctrl.get_selection())", type(ctrl.get_selection()) if isinstance(ctrl.get_selection(), list): value = ctrl.get_selection() if not value: display( win, "%s field is empty or the items are not checked!" % ctrl.get_name(), False) return else: value = str(ctrl.get_selection()) if len(value) == 0: display(win, "%s field is empty!" % ctrl.get_name()) return if '/' in value and '$' not in value and not isinstance( value, list): if not os.path.exists(ctrl.get_selection()): display( win, "%s field contains incorrect path. Please update the path!" % ctrl.get_name()) return config_list.append(ctrl) # Get the user's CPAC output directory for use in this script for config in config_list: #print config.get_name(), " ", config.get_selection() if config.get_name() == 'outputDirectory': outDir = config.get_selection() # Write out a pipeline_config file, read it in and then delete it # (Will revise the data structure of the config files later so this # can just pass the data structure instead of doing it this way) try: self.write(outDir + 'testConfig.yml', config_list) c = Configuration( yaml.load( open(os.path.realpath(outDir + 'testConfig.yml'), 'r'))) os.remove(outDir + 'testConfig.yml') except: errDlg2 = wx.MessageDialog( self, 'A problem occurred with preparing the pipeline test run. \n\n' \ 'Please ensure you have rights access to the directories you' \ ' have chosen for the CPAC working, crash, and output folders.', 'Test Configuration Error', wx.OK | wx.ICON_ERROR) errDlg2.ShowModal() errDlg2.Destroy() if (1 in c.runNuisance) or (c.Corrections != None): strategies = sorted(build_strategies(c)) else: strategies = None # Run the actual pipeline building prep and see if it works or not try: testDlg1 = wx.MessageDialog( self, 'Click OK to run the test. This should take only a few seconds.', 'Running Test', wx.OK | wx.ICON_INFORMATION) testDlg1.ShowModal() prep_workflow(sublist[0], c, strategies, 0) except: testDlg1.Destroy() errDlg1 = wx.MessageDialog( self, 'There are issues with the current configuration which need to be' \ ' resolved - please check to make sure the options you are running' \ ' have the proper pre-requisites selected.', 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlg1.ShowModal() errDlg1.Destroy() else: testDlg1.Destroy() okDlg1 = wx.MessageDialog( self, 'The current configuration will run successfully. You can safely' \ ' save and run this setup!', 'Pipeline Ready', wx.OK | wx.ICON_INFORMATION) okDlg1.ShowModal() okDlg1.Destroy()
def testConfig(self, event): ''' This function runs when the user clicks the "Test Configuration" button in the pipeline configuration window. It prompts the user for a sample subject list (i.e. one that they will be using with the config they are building). Then it builds the pipeline but does not run it. It then reports whether or not the config will run or not depending on if the pipeline gets built successfully. ''' import os import yaml from CPAC.utils import Configuration from CPAC.pipeline.cpac_pipeline import prep_workflow from CPAC.pipeline.cpac_runner import build_strategies def display(win, msg, changeBg=True): wx.MessageBox(msg, "Error") if changeBg: win.SetBackgroundColour("pink") win.SetFocus() win.Refresh() # Collect a sample subject list and parse it in testDlg0 = wx.MessageDialog( self, 'This tool will run a quick check on the current pipeline configuration.' \ ' Click OK to provide a subject list you will be using with this setup.', 'Subject List', wx.OK | wx.ICON_INFORMATION) testDlg0.ShowModal() testDlg0.Destroy() dlg = wx.FileDialog(self, message="Choose the CPAC Subject list file", defaultDir=os.getcwd(), defaultFile="CPAC_subject_list.yml", wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml", style=wx.OPEN | wx.CHANGE_DIR) if dlg.ShowModal() == wx.ID_OK: subListPath = dlg.GetPath() # Load and test the subject list sublist = yaml.load(open(os.path.realpath(subListPath), 'r')) sub_flg = self.test_sublist(sublist) if not sub_flg: raise Exception # Following code reads in the parameters and selections from the # pipeline configuration window and populate the config_list config_list = [] wf_counter = [] for page in self.nb.get_page_list(): switch = page.page.get_switch() ctrl_list = page.page.get_ctrl_list() validate = False if switch: switch_val = str(switch.get_selection()).lower() if switch_val == 'on' or switch_val == 'true' or switch_val == '1': validate = True wf_counter.append(page.get_counter()) for ctrl in ctrl_list: # option_name will be the selection name as it is written # as the dictionary key of the config.yml dictionary option_name = ctrl.get_name() #validating if (switch == None or validate) and ctrl.get_validation() \ and (option_name != 'derivativeList') and (option_name != 'modelConfigs'): win = ctrl.get_ctrl() if isinstance(ctrl.get_selection(), list): value = ctrl.get_selection() if not value: display( win, "%s field is empty or the items are not checked!" % ctrl.get_name(), False) return else: value = str(ctrl.get_selection()) if len(value) == 0: display(win, "%s field is empty!" % ctrl.get_name()) return if '/' in value and '$' not in value and not isinstance( value, list): if not os.path.exists( ctrl.get_selection()) and value != 'On/Off': display( win, "%s field contains incorrect path. Please update the path!" % ctrl.get_name()) return config_list.append(ctrl) # Get the user's CPAC output directory for use in this script for config in config_list: if config.get_name() == 'outputDirectory': outDir = config.get_selection() # Write out a pipeline_config file, read it in and then delete it # (Will revise the data structure of the config files later so this # can just pass the data structure instead of doing it this way) try: self.write(outDir + 'testConfig.yml', config_list) c = Configuration( yaml.load( open(os.path.realpath(outDir + 'testConfig.yml'), 'r'))) os.remove(outDir + 'testConfig.yml') except: errDlg2 = wx.MessageDialog( self, 'A problem occurred with preparing the pipeline test run. \n\n' \ 'Please ensure you have rights access to the directories you' \ ' have chosen for the CPAC working, crash, and output folders.', 'Test Configuration Error', wx.OK | wx.ICON_ERROR) errDlg2.ShowModal() errDlg2.Destroy() if (1 in c.runNuisance) or (c.Corrections != None): strategies = sorted(build_strategies(c)) else: strategies = None # Run the actual pipeline building prep and see if it works or not testDlg1 = wx.MessageDialog( self, 'Click OK to run the test. This should take only a few seconds.', 'Running Test', wx.OK | wx.ICON_INFORMATION) testDlg1.ShowModal() # Check file paths first # Just getting proper names of config file parameters try: params_file = open( p.resource_filename('CPAC', 'GUI/resources/config_parameters.txt'), "r") except: print "Error: Could not open configuration parameter file.", "\n" raise Exception paramInfo = params_file.read().split('\n') paramList = [] for param in paramInfo: if param != '': paramList.append(param.split(',')) # function for file path checking def testFile(filepath, paramName, switch): try: if (1 in switch) and (filepath != None): fileTest = open(filepath) fileTest.close() except: testDlg1.Destroy() for param in paramList: if param[0] == paramName: paramTitle = param[1] paramGroup = param[2] break errDlgFileTest = wx.MessageDialog( self, 'Error reading file - either it does not exist or you' \ ' do not have read access. \n\n' \ 'Parameter: %s \n' \ 'In tab: %s \n\n' \ 'Path: %s' % (paramTitle, paramGroup, filepath), 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlgFileTest.ShowModal() errDlgFileTest.Destroy() testFile(c.template_brain_only_for_anat, 'template_brain_only_for_anat', c.runRegistrationPreprocessing) testFile(c.template_skull_for_anat, 'template_skull_for_anat', c.runRegistrationPreprocessing) testFile(c.PRIORS_WHITE, 'PRIORS_WHITE', c.runSegmentationPreprocessing) testFile(c.PRIORS_GRAY, 'PRIORS_GRAY', c.runSegmentationPreprocessing) testFile(c.PRIORS_CSF, 'PRIORS_CSF', c.runSegmentationPreprocessing) testFile(c.template_brain_only_for_func, 'template_brain_only_for_func', c.runRegisterFuncToMNI) testFile(c.template_skull_for_func, 'template_skull_for_func', c.runRegisterFuncToMNI) testFile(c.identityMatrix, 'identityMatrix', c.runRegisterFuncToMNI) testFile(c.boundaryBasedRegistrationSchedule, 'boundaryBasedRegistrationSchedule', c.runRegisterFuncToAnat) testFile(c.lateral_ventricles_mask, 'lateral_ventricles_mask', c.runNuisance) testFile(c.seedSpecificationFile, 'seedSpecificationFile', [1]) testFile(c.roiSpecificationFile, 'roiSpecificationFile', c.runROITimeseries) testFile(c.roiSpecificationFileForSCA, 'roiSpecificationFileForSCA', c.runROITimeseries) testFile(c.maskSpecificationFile, 'maskSpecificationFile', c.runVoxelTimeseries) testFile(c.maskSpecificationFileForSCA, 'maskSpecificationFileForSCA', c.runVoxelTimeseries) testFile(c.spatialPatternMaps, 'spatialPatternMaps', c.runSpatialRegression) testFile(c.template_symmetric_brain_only, 'template_symmetric_brain_only', c.runVMHC) testFile(c.template_symmetric_skull, 'template_symmetric_skull', c.runVMHC) testFile(c.dilated_symmetric_brain_mask, 'dilated_symmetric_brain_mask', c.runVMHC) testFile(c.configFileTwomm, 'configFileTwomm', c.runVMHC) testFile(c.templateSpecificationFile, 'templateSpecificationFile', c.runNetworkCentrality) testFile(c.bascAffinityThresholdFile, 'bascAffinityThresholdFile', c.runBASC) testFile(c.cwasROIFile, 'cwasROIFile', c.runCWAS) testFile(c.cwasRegressorFile, 'cwasRegressorFile', c.runCWAS) try: # Run the pipeline building prep_workflow(sublist[0], c, strategies, 0) except Exception as xxx: print xxx print "an exception occured" testDlg1.Destroy() errDlg1 = wx.MessageDialog( self, 'There are issues with the current configuration ' \ 'which need to be resolved - please check to make ' \ 'sure the options you are running have the proper ' \ 'pre-requisites selected.\n\nIssue Info:\n%s' % xxx, 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlg1.ShowModal() errDlg1.Destroy() else: testDlg1.Destroy() okDlg1 = wx.MessageDialog( self, 'The current configuration will run successfully. You can safely' \ ' save and run this setup!', 'Pipeline Ready', wx.OK | wx.ICON_INFORMATION) okDlg1.ShowModal() okDlg1.Destroy()
def testConfig(self, event): ''' This function runs when the user clicks the "Test Configuration" button in the pipeline configuration window. It prompts the user for a sample subject list (i.e. one that they will be using with the config they are building). Then it builds the pipeline but does not run it. It then reports whether or not the config will run or not depending on if the pipeline gets built successfully. ''' import os import yaml from CPAC.utils import Configuration from CPAC.pipeline.cpac_pipeline import prep_workflow from CPAC.pipeline.cpac_runner import build_strategies def display(win, msg, changeBg=True): wx.MessageBox(msg, "Error") if changeBg: win.SetBackgroundColour("pink") win.SetFocus() win.Refresh() # Collect a sample subject list and parse it in testDlg0 = wx.MessageDialog( self, 'This tool will run a quick check on the current pipeline configuration.' \ ' Click OK to provide a subject list you will be using with this setup.', 'Subject List', wx.OK | wx.ICON_INFORMATION) testDlg0.ShowModal() testDlg0.Destroy() dlg = wx.FileDialog( self, message="Choose the CPAC Subject list file", defaultDir=os.getcwd(), defaultFile="CPAC_subject_list.yml", wildcard="YAML files(*.yaml, *.yml)|*.yaml;*.yml", style=wx.OPEN | wx.CHANGE_DIR) if dlg.ShowModal() == wx.ID_OK: subListPath = dlg.GetPath() sublist = yaml.load(open(os.path.realpath(subListPath), 'r')) # Check to ensure the user is providing an actual subject # list and not some other kind of file try: subInfo = sublist[0] except: errDlg4 = wx.MessageDialog( self, 'ERROR: Subject list file not in proper format - check if you' \ ' loaded the correct file? \n\n' \ 'Error name: config_window_0001', 'Subject List Error', wx.OK | wx.ICON_ERROR) errDlg4.ShowModal() errDlg4.Destroy() raise Exception # Another check to ensure the actual subject list was generated # properly and that it will work if 'subject_id' not in subInfo: errDlg3 = wx.MessageDialog( self, 'ERROR: Subject list file not in proper format - check if you' \ ' loaded the correct file? \n\n' \ 'Error name: config_window_0002', 'Subject List Error', wx.OK | wx.ICON_ERROR) errDlg3.ShowModal() errDlg3.Destroy() raise Exception # Following code reads in the parameters and selections from the # pipeline configuration window and populate the config_list config_list = [] wf_counter = [] #print "self.nb.get_page_list()", self.nb.get_page_list() for page in self.nb.get_page_list(): #print "page ----> ", page switch = page.page.get_switch() #print "switch ---->", switch ctrl_list = page.page.get_ctrl_list() validate = False if switch: switch_val = str(switch.get_selection()).lower() #print "switch_val ---->", switch_val if switch_val == 'on' or switch_val == 'true' or switch_val == '1': validate = True wf_counter.append(page.get_counter()) for ctrl in ctrl_list: #validating if (switch == None or validate) and ctrl.get_validation(): win = ctrl.get_ctrl() #print "validating ctrl-->", ctrl.get_name() #print "ctrl.get_selection()", ctrl.get_selection() #print "type(ctrl.get_selection())", type(ctrl.get_selection()) if isinstance(ctrl.get_selection(), list): value = ctrl.get_selection() if not value: display( win, "%s field is empty or the items are not checked!" % ctrl.get_name(), False) return else: value = str(ctrl.get_selection()) if len(value) == 0: display(win, "%s field is empty!" % ctrl.get_name()) return if '/' in value and '$' not in value and not isinstance(value, list): if not os.path.exists(ctrl.get_selection()): display( win, "%s field contains incorrect path. Please update the path!" % ctrl.get_name()) return config_list.append(ctrl) # Get the user's CPAC output directory for use in this script for config in config_list: #print config.get_name(), " ", config.get_selection() if config.get_name() == 'outputDirectory': outDir = config.get_selection() # Write out a pipeline_config file, read it in and then delete it # (Will revise the data structure of the config files later so this # can just pass the data structure instead of doing it this way) try: self.write(outDir + 'testConfig.yml', config_list) c = Configuration(yaml.load(open(os.path.realpath(outDir + 'testConfig.yml'), 'r'))) os.remove(outDir + 'testConfig.yml') except: errDlg2 = wx.MessageDialog( self, 'A problem occurred with preparing the pipeline test run. \n\n' \ 'Please ensure you have rights access to the directories you' \ ' have chosen for the CPAC working, crash, and output folders.', 'Test Configuration Error', wx.OK | wx.ICON_ERROR) errDlg2.ShowModal() errDlg2.Destroy() if (1 in c.runNuisance) or (c.Corrections != None): strategies = sorted(build_strategies(c)) else: strategies = None # Run the actual pipeline building prep and see if it works or not try: testDlg1 = wx.MessageDialog( self, 'Click OK to run the test. This should take only a few seconds.', 'Running Test', wx.OK | wx.ICON_INFORMATION) testDlg1.ShowModal() prep_workflow(sublist[0], c, strategies, 0) except: testDlg1.Destroy() errDlg1 = wx.MessageDialog( self, 'There are issues with the current configuration which need to be' \ ' resolved - please check to make sure the options you are running' \ ' have the proper pre-requisites selected.', 'Pipeline Not Ready', wx.OK | wx.ICON_ERROR) errDlg1.ShowModal() errDlg1.Destroy() else: testDlg1.Destroy() okDlg1 = wx.MessageDialog( self, 'The current configuration will run successfully. You can safely' \ ' save and run this setup!', 'Pipeline Ready', wx.OK | wx.ICON_INFORMATION) okDlg1.ShowModal() okDlg1.Destroy()