def run(config_file, output_path_file):
import re
import os
import glob
path, fname = os.path.split(os.path.realpath(config_file))
sys.path.append(path)
c = __import__(fname.split('.')[0])
subject_paths = []
input_subject_ids = [subject_id.rstrip('\r\n') for subject_id in open(c.groupAnalysisSubjectList, 'r').readlines()]
for file in glob.glob(os.path.abspath(output_path_file)):
path_list = open(file, 'r').readlines()
subject_paths.extend([s.rstrip('\r\n') for s in path_list])
set_subject_paths = set(subject_paths)
subject_paths = list(set_subject_paths)
#base_path = os.path.dirname(os.path.commonprefix(subject_paths))
base_path = c.sinkDirectory
from collections import defaultdict
analysis_map = defaultdict(list)
analysis_map_gp = defaultdict(list)
for subject_path in subject_paths:
#Remove the base bath offset
rs_path = subject_path.replace(base_path, "", 1)
rs_path = rs_path.lstrip('/')
folders = split_folders(rs_path)
pipeline_id = folders[0]
subject_id = folders[1]
resource_id = folders[2]
scan_id = folders[3]
key = subject_path.replace(subject_id, '*')
analysis_map[(resource_id, key)].append((pipeline_id, subject_id, scan_id, subject_path))
# separate map for group analysis
if c.mixedScanAnalysis == True:
key = key.replace(scan_id, '*')
if subject_id in input_subject_ids:
analysis_map_gp[(resource_id, key)].append((pipeline_id, subject_id, scan_id, subject_path))
for resource, glob_key in analysis_map.keys():
if resource == 'functional_mni':
if 1 in c.runBASC:
if not c.runOnGrid:
from CPAC.pipeline.cpac_basc_pipeline import prep_basc_workflow
prep_basc_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, args.config, resource, analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, args.config, resource, analysis_map[(resource, glob_key)])
if 1 in c.runCWAS:
if not c.runOnGrid:
from CPAC.pipeline.cpac_cwas_pipeline import prep_cwas_workflow
prep_cwas_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, args.config, resource, analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, args.config, resource, analysis_map[(resource, glob_key)])
for resource, glob_key in analysis_map_gp.keys():
if resource in c.derivativeList:
if 1 in c.runGroupAnalysis:
if not c.runOnGrid:
from CPAC.pipeline.cpac_group_analysis_pipeline import prep_group_analysis_workflow
prep_group_analysis_workflow(c, resource, analysis_map_gp[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, args.config, resource, analysis_map_gp[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, args.config, resource, analysis_map_gp[(resource, glob_key)])
def run(config_file, output_path_file):
# Runs group analysis
import yaml
# Load the config file into 'c'
c = Configuration(yaml.load(open(os.path.realpath(config_file), 'r')))
subject_paths = []
for file in glob.glob(os.path.abspath(output_path_file)):
path_list = open(file, 'r').readlines()
subject_paths.extend([s.rstrip('\r\n') for s in path_list])
set_subject_paths = set(subject_paths)
subject_paths = list(set_subject_paths)
#base_path = os.path.dirname(os.path.commonprefix(subject_paths))
base_path = c.outputDirectory
from collections import defaultdict
analysis_map = defaultdict(list)
analysis_map_gp = defaultdict(list)
for subject_path in subject_paths:
#Remove the base bath offset
rs_path = subject_path.replace(base_path, "", 1)
rs_path = rs_path.lstrip('/')
folders = split_folders(rs_path)
pipeline_id = folders[0]
subject_id = folders[1]
resource_id = folders[2]
scan_id = folders[3]
#if scan_id == '_scan_rest_1_rest':
key = subject_path.replace(subject_id, '*')
analysis_map[(resource_id, key)].append((pipeline_id, subject_id, scan_id, subject_path))
# separate map for group analysis
#if c.mixedScanAnalysis == True:
# key = key.replace(scan_id, '*')
analysis_map_gp[(resource_id, key)].append((pipeline_id, subject_id, scan_id, subject_path))
gpa_start_datetime = strftime("%Y-%m-%d %H:%M:%S")
gpa_starttime_string = gpa_start_datetime.replace(' ','_')
gpa_starttime_string = gpa_starttime_string.replace(':','-')
timing = open(os.path.join(c.outputDirectory, 'group_analysis_timing_%s_%s.txt' % (c.pipelineName, gpa_starttime_string)), 'wt')
sca_roi_runs = 0
sca_roi_time = 0
sca_seed_runs = 0
sca_seed_time = 0
sca_tempreg_runs = 0
sca_tempreg_time = 0
dr_tempreg_runs = 0
dr_tempreg_time = 0
vmhc_z_runs = 0
vmhc_z_time = 0
alff_Z_runs = 0
alff_Z_time = 0
falff_Z_runs = 0
falff_Z_time = 0
reho_Z_runs = 0
reho_Z_time = 0
centrality_outputs_runs = 0
centrality_outputs_time = 0
# Start timing here
gpa_start_time = time.time()
for resource, glob_key in analysis_map.keys():
if resource == 'functional_mni':
wf_start_time = time.time()
if 1 in c.runBASC:
if not c.runOnGrid:
from CPAC.pipeline.cpac_basc_pipeline import prep_basc_workflow
prep_basc_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
if 1 in c.runCWAS:
if not c.runOnGrid:
from CPAC.pipeline.cpac_cwas_pipeline import prep_cwas_workflow
prep_cwas_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
print >>timing, "Group analysis workflow completed for resource: ", resource
print >>timing, "Elapsed run time (minutes): ", ((time.time() - wf_start_time)/60)
print >>timing, ""
for resource, glob_key in analysis_map_gp.keys():
if resource in c.derivativeList:
wf_start_time = time.time()
if 1 in c.runGroupAnalysis:
#get all the motion parameters across subjects
try:
from CPAC.utils import extract_parameters
extract_parameters.run(c.outputDirectory)
except Exception:
print "Extract parameters script did not run correctly"
if not c.runOnGrid:
from CPAC.pipeline.cpac_group_analysis_pipeline import prep_group_analysis_workflow
#procss = Process(target=prep_group_analysis_workflow, args=(c, resource, analysis_map_gp[(resource, glob_key)]))
#print c, " ", resource, " ", analysis_map_gp[(resource, glob_key)], " ", glob_key
prep_group_analysis_workflow(c, resource, analysis_map_gp[(resource, glob_key)])
if c.runOnGrid:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource, analysis_map_gp[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource, analysis_map_gp[(resource, glob_key)])
print >>timing, "Group analysis workflow completed for resource: ", resource
print >>timing, "Elapsed run time (minutes): ", ((time.time() - wf_start_time)/60)
print >>timing, ""
# This can be implemented more sleekly using a dictionary, have to do this at some point
if resource == 'sca_roi_Z_to_standard_smooth':
sca_roi_runs += 1
sca_roi_time = sca_roi_time + ((time.time() - wf_start_time)/60)
elif resource == 'sca_seed_Z_to_standard_smooth':
sca_seed_runs += 1
sca_seed_time = sca_seed_time + ((time.time() - wf_start_time)/60)
elif resource == 'sca_tempreg_maps_z_files_smooth':
sca_tempreg_runs += 1
sca_tempreg_time = sca_tempreg_time + ((time.time() - wf_start_time)/60)
elif resource == 'dr_tempreg_maps_z_files_smooth':
dr_tempreg_runs += 1
dr_tempreg_time = dr_tempreg_time + ((time.time() - wf_start_time)/60)
elif resource == 'vmhc_z_score_stat_map':
vmhc_z_runs += 1
vmhc_z_time = vmhc_z_time + ((time.time() - wf_start_time)/60)
elif resource == 'alff_Z_to_standard_smooth':
alff_Z_runs += 1
alff_Z_time = alff_Z_time + ((time.time() - wf_start_time)/60)
elif resource == 'falff_Z_to_standard_smooth':
falff_Z_runs += 1
falff_Z_time = falff_Z_time + ((time.time() - wf_start_time)/60)
elif resource == 'reho_Z_to_standard_smooth':
reho_Z_runs += 1
reho_Z_time = reho_Z_time + ((time.time() - wf_start_time)/60)
elif resource == 'centrality_outputs_smoothed':
centrality_outputs_runs += 1
centrality_outputs_time = centrality_outputs_time + ((time.time() - wf_start_time)/60)
'''
procss = []
for resource, glob_key in analysis_map_gp.keys():
if resource in c.derivativeList:
if 1 in c.runGroupAnalysis:
#get all the motion parameters across subjects
try:
from CPAC.utils import extract_parameters
extract_parameters.run(c.outputDirectory)
except Exception:
print "Extract parameters script did not run correctly"
if not c.runOnGrid:
from CPAC.pipeline.cpac_group_analysis_pipeline import prep_group_analysis_workflow
procss.append(Process(target=prep_group_analysis_workflow, args=(c, resource, analysis_map_gp[(resource, glob_key)])))
pid = open(os.path.join(c.outputDirectory, 'pid_group.txt'), 'w')
jobQueue = []
if len(c.derivativeList) <= c.numSubjectsAtOnce:
"""
Stream all the subjects as sublist is
less than or equal to the number of
subjects that need to run
"""
for p in procss:
p.start()
print >>pid,p.pid
else:
"""
Stream the subject workflows for preprocessing.
At Any time in the pipeline c.numSubjectsAtOnce
will run, unless the number remaining is less than
the value of the parameter stated above
"""
idx = 0
while(idx < len(c.derivativeList)):
if len(jobQueue) == 0 and idx == 0:
idc = idx
for p in procss[idc: idc + c.numSubjectsAtOnce]:
p.start()
print >>pid,p.pid
jobQueue.append(p)
idx += 1
else:
for job in jobQueue:
if not job.is_alive():
print 'found dead job ', job
loc = jobQueue.index(job)
del jobQueue[loc]
procss[idx].start()
jobQueue.append(procss[idx])
idx += 1
pid.close()
'''
print >>timing, "Entire group analysis run complete."
print >>timing, "Elapsed run time (minutes): ", ((time.time() - gpa_start_time)/60)
print >>timing, ""
print >>timing, "sca_roi_Z_to_standard_smooth"
print >>timing, "Number of runs: ", sca_roi_runs
print >>timing, "Total run time (minutes): ", sca_roi_time
print >>timing, ""
print >>timing, "sca_seed_Z_to_standard_smooth"
print >>timing, "Number of runs: ", sca_seed_runs
print >>timing, "Total run time (minutes): ", sca_seed_time
print >>timing, ""
print >>timing, "sca_tempreg_maps_z_files_smooth"
print >>timing, "Number of runs: ", sca_tempreg_runs
print >>timing, "Total run time (minutes): ", sca_tempreg_time
print >>timing, ""
print >>timing, "dr_tempreg_maps_z_files_smooth"
print >>timing, "Number of runs: ", dr_tempreg_runs
print >>timing, "Total run time (minutes): ", dr_tempreg_time
print >>timing, ""
print >>timing, "vmhc_z_score_stat_map"
print >>timing, "Number of runs: ", vmhc_z_runs
print >>timing, "Total run time (minutes): ", vmhc_z_time
print >>timing, ""
print >>timing, "alff_Z_to_standard_smooth"
print >>timing, "Number of runs: ", alff_Z_runs
print >>timing, "Total run time (minutes): ", alff_Z_time
print >>timing, ""
print >>timing, "falff_Z_to_standard_smooth"
print >>timing, "Number of runs: ", falff_Z_runs
print >>timing, "Total run time (minutes): ", falff_Z_time
print >>timing, ""
print >>timing, "reho_Z_to_standard_smooth"
print >>timing, "Number of runs: ", reho_Z_runs
print >>timing, "Total run time (minutes): ", reho_Z_time
print >>timing, ""
print >>timing, "centrality_outputs_smoothed"
print >>timing, "Number of runs: ", centrality_outputs_runs
print >>timing, "Total run time (minutes): ", centrality_outputs_time
print >>timing, ""
timing.close()
def run(config_file, output_path_file):
# Runs group analysis
import yaml
# Load the config file into 'c'
c = Configuration(yaml.load(open(os.path.realpath(config_file), 'r')))
subject_paths = []
for file in glob.glob(os.path.abspath(output_path_file)):
path_list = open(file, 'r').readlines()
subject_paths.extend([s.rstrip('\r\n') for s in path_list])
set_subject_paths = set(subject_paths)
subject_paths = list(set_subject_paths)
#base_path = os.path.dirname(os.path.commonprefix(subject_paths))
base_path = c.outputDirectory
from collections import defaultdict
analysis_map = defaultdict(list)
analysis_map_gp = defaultdict(list)
for subject_path in subject_paths:
#Remove the base bath offset
rs_path = subject_path.replace(base_path, "", 1)
rs_path = rs_path.lstrip('/')
folders = split_folders(rs_path)
pipeline_id = folders[0]
subject_id = folders[1]
resource_id = folders[2]
scan_id = folders[3]
#if scan_id == '_scan_rest_1_rest':
key = subject_path.replace(subject_id, '*')
analysis_map[(resource_id, key)].append(
(pipeline_id, subject_id, scan_id, subject_path))
# separate map for group analysis
#if c.mixedScanAnalysis == True:
# key = key.replace(scan_id, '*')
analysis_map_gp[(resource_id, key)].append(
(pipeline_id, subject_id, scan_id, subject_path))
gpa_start_datetime = strftime("%Y-%m-%d %H:%M:%S")
gpa_starttime_string = gpa_start_datetime.replace(' ', '_')
gpa_starttime_string = gpa_starttime_string.replace(':', '-')
timing = open(
os.path.join(
c.outputDirectory, 'group_analysis_timing_%s_%s.txt' %
(c.pipelineName, gpa_starttime_string)), 'wt')
sca_roi_runs = 0
sca_roi_time = 0
sca_seed_runs = 0
sca_seed_time = 0
sca_tempreg_runs = 0
sca_tempreg_time = 0
dr_tempreg_runs = 0
dr_tempreg_time = 0
vmhc_z_runs = 0
vmhc_z_time = 0
alff_Z_runs = 0
alff_Z_time = 0
falff_Z_runs = 0
falff_Z_time = 0
reho_Z_runs = 0
reho_Z_time = 0
centrality_outputs_runs = 0
centrality_outputs_time = 0
# Start timing here
gpa_start_time = time.time()
for resource, glob_key in analysis_map.keys():
if resource == 'functional_mni':
wf_start_time = time.time()
if 1 in c.runBASC:
if not c.runOnGrid:
from CPAC.pipeline.cpac_basc_pipeline import prep_basc_workflow
prep_basc_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource,
analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource,
analysis_map[(resource, glob_key)])
if 1 in c.runCWAS:
if not c.runOnGrid:
from CPAC.pipeline.cpac_cwas_pipeline import prep_cwas_workflow
prep_cwas_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource,
analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource,
analysis_map[(resource, glob_key)])
print >> timing, "Group analysis workflow completed for resource: ", resource
print >> timing, "Elapsed run time (minutes): ", (
(time.time() - wf_start_time) / 60)
print >> timing, ""
for resource, glob_key in analysis_map_gp.keys():
if resource in c.derivativeList:
wf_start_time = time.time()
if 1 in c.runGroupAnalysis:
#get all the motion parameters across subjects
try:
from CPAC.utils import extract_parameters
extract_parameters.run(c.outputDirectory)
except Exception:
print "Extract parameters script did not run correctly"
if not c.runOnGrid:
from CPAC.pipeline.cpac_group_analysis_pipeline import prep_group_analysis_workflow
#procss = Process(target=prep_group_analysis_workflow, args=(c, resource, analysis_map_gp[(resource, glob_key)]))
#print c, " ", resource, " ", analysis_map_gp[(resource, glob_key)], " ", glob_key
prep_group_analysis_workflow(
c, resource, analysis_map_gp[(resource, glob_key)])
if c.runOnGrid:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource,
analysis_map_gp[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource,
analysis_map_gp[(resource, glob_key)])
print >> timing, "Group analysis workflow completed for resource: ", resource
print >> timing, "Elapsed run time (minutes): ", (
(time.time() - wf_start_time) / 60)
print >> timing, ""
# This can be implemented more sleekly using a dictionary, have to do this at some point
if resource == 'sca_roi_Z_to_standard_smooth':
sca_roi_runs += 1
sca_roi_time = sca_roi_time + (
(time.time() - wf_start_time) / 60)
elif resource == 'sca_seed_Z_to_standard_smooth':
sca_seed_runs += 1
sca_seed_time = sca_seed_time + (
(time.time() - wf_start_time) / 60)
elif resource == 'sca_tempreg_maps_z_files_smooth':
sca_tempreg_runs += 1
sca_tempreg_time = sca_tempreg_time + (
(time.time() - wf_start_time) / 60)
elif resource == 'dr_tempreg_maps_z_files_smooth':
dr_tempreg_runs += 1
dr_tempreg_time = dr_tempreg_time + (
(time.time() - wf_start_time) / 60)
elif resource == 'vmhc_z_score_stat_map':
vmhc_z_runs += 1
vmhc_z_time = vmhc_z_time + (
(time.time() - wf_start_time) / 60)
elif resource == 'alff_Z_to_standard_smooth':
alff_Z_runs += 1
alff_Z_time = alff_Z_time + (
(time.time() - wf_start_time) / 60)
elif resource == 'falff_Z_to_standard_smooth':
falff_Z_runs += 1
falff_Z_time = falff_Z_time + (
(time.time() - wf_start_time) / 60)
elif resource == 'reho_Z_to_standard_smooth':
reho_Z_runs += 1
reho_Z_time = reho_Z_time + (
(time.time() - wf_start_time) / 60)
elif resource == 'centrality_outputs_smoothed':
centrality_outputs_runs += 1
centrality_outputs_time = centrality_outputs_time + (
(time.time() - wf_start_time) / 60)
'''
procss = []
for resource, glob_key in analysis_map_gp.keys():
if resource in c.derivativeList:
if 1 in c.runGroupAnalysis:
#get all the motion parameters across subjects
try:
from CPAC.utils import extract_parameters
extract_parameters.run(c.outputDirectory)
except Exception:
print "Extract parameters script did not run correctly"
if not c.runOnGrid:
from CPAC.pipeline.cpac_group_analysis_pipeline import prep_group_analysis_workflow
procss.append(Process(target=prep_group_analysis_workflow, args=(c, resource, analysis_map_gp[(resource, glob_key)])))
pid = open(os.path.join(c.outputDirectory, 'pid_group.txt'), 'w')
jobQueue = []
if len(c.derivativeList) <= c.numSubjectsAtOnce:
"""
Stream all the subjects as sublist is
less than or equal to the number of
subjects that need to run
"""
for p in procss:
p.start()
print >>pid,p.pid
else:
"""
Stream the subject workflows for preprocessing.
At Any time in the pipeline c.numSubjectsAtOnce
will run, unless the number remaining is less than
the value of the parameter stated above
"""
idx = 0
while(idx < len(c.derivativeList)):
if len(jobQueue) == 0 and idx == 0:
idc = idx
for p in procss[idc: idc + c.numSubjectsAtOnce]:
p.start()
print >>pid,p.pid
jobQueue.append(p)
idx += 1
else:
for job in jobQueue:
if not job.is_alive():
print 'found dead job ', job
loc = jobQueue.index(job)
del jobQueue[loc]
procss[idx].start()
jobQueue.append(procss[idx])
idx += 1
pid.close()
'''
print >> timing, "Entire group analysis run complete."
print >> timing, "Elapsed run time (minutes): ", (
(time.time() - gpa_start_time) / 60)
print >> timing, ""
print >> timing, "sca_roi_Z_to_standard_smooth"
print >> timing, "Number of runs: ", sca_roi_runs
print >> timing, "Total run time (minutes): ", sca_roi_time
print >> timing, ""
print >> timing, "sca_seed_Z_to_standard_smooth"
print >> timing, "Number of runs: ", sca_seed_runs
print >> timing, "Total run time (minutes): ", sca_seed_time
print >> timing, ""
print >> timing, "sca_tempreg_maps_z_files_smooth"
print >> timing, "Number of runs: ", sca_tempreg_runs
print >> timing, "Total run time (minutes): ", sca_tempreg_time
print >> timing, ""
print >> timing, "dr_tempreg_maps_z_files_smooth"
print >> timing, "Number of runs: ", dr_tempreg_runs
print >> timing, "Total run time (minutes): ", dr_tempreg_time
print >> timing, ""
print >> timing, "vmhc_z_score_stat_map"
print >> timing, "Number of runs: ", vmhc_z_runs
print >> timing, "Total run time (minutes): ", vmhc_z_time
print >> timing, ""
print >> timing, "alff_Z_to_standard_smooth"
print >> timing, "Number of runs: ", alff_Z_runs
print >> timing, "Total run time (minutes): ", alff_Z_time
print >> timing, ""
print >> timing, "falff_Z_to_standard_smooth"
print >> timing, "Number of runs: ", falff_Z_runs
print >> timing, "Total run time (minutes): ", falff_Z_time
print >> timing, ""
print >> timing, "reho_Z_to_standard_smooth"
print >> timing, "Number of runs: ", reho_Z_runs
print >> timing, "Total run time (minutes): ", reho_Z_time
print >> timing, ""
print >> timing, "centrality_outputs_smoothed"
print >> timing, "Number of runs: ", centrality_outputs_runs
print >> timing, "Total run time (minutes): ", centrality_outputs_time
print >> timing, ""
timing.close()
def run(config_file, output_path_file):
# Runs group analysis
import re
import os
import glob
import yaml
# Load the config file into 'c'
c = Configuration(yaml.load(open(os.path.realpath(config_file), 'r')))
subject_paths = []
for file in glob.glob(os.path.abspath(output_path_file)):
path_list = open(file, 'r').readlines()
subject_paths.extend([s.rstrip('\r\n') for s in path_list])
set_subject_paths = set(subject_paths)
subject_paths = list(set_subject_paths)
#base_path = os.path.dirname(os.path.commonprefix(subject_paths))
base_path = c.outputDirectory
from collections import defaultdict
analysis_map = defaultdict(list)
analysis_map_gp = defaultdict(list)
for subject_path in subject_paths:
#Remove the base bath offset
rs_path = subject_path.replace(base_path, "", 1)
rs_path = rs_path.lstrip('/')
folders = split_folders(rs_path)
pipeline_id = folders[0]
subject_id = folders[1]
resource_id = folders[2]
scan_id = folders[3]
key = subject_path.replace(subject_id, '*')
analysis_map[(resource_id, key)].append(
(pipeline_id, subject_id, scan_id, subject_path))
# separate map for group analysis
# if c.mixedScanAnalysis == True:
# key = key.replace(scan_id, '*')
analysis_map_gp[(resource_id, key)].append(
(pipeline_id, subject_id, scan_id, subject_path))
for resource, glob_key in analysis_map.keys():
if resource == 'functional_mni':
if 1 in c.runBASC:
if not c.runOnGrid:
from CPAC.pipeline.cpac_basc_pipeline import prep_basc_workflow
prep_basc_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource,
analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource,
analysis_map[(resource, glob_key)])
if 1 in c.runCWAS:
if not c.runOnGrid:
from CPAC.pipeline.cpac_cwas_pipeline import prep_cwas_workflow
prep_cwas_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource,
analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource,
analysis_map[(resource, glob_key)])
for resource, glob_key in analysis_map_gp.keys():
if resource in c.derivativeList:
if 1 in c.runGroupAnalysis:
#get all the motion parameters across subjects
try:
from CPAC.utils import extract_parameters
extract_parameters.run(c.outputDirectory)
except Exception:
print "Extract parameters script did not run correctly"
if not c.runOnGrid:
from CPAC.pipeline.cpac_group_analysis_pipeline import prep_group_analysis_workflow
#print c, " ", resource, " ", analysis_map_gp[(resource, glob_key)], " ", glob_key
prep_group_analysis_workflow(
c, resource, analysis_map_gp[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource,
analysis_map_gp[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource,
analysis_map_gp[(resource, glob_key)])
def run(config_file, output_path_file):
# Runs group analysis
import re
import os
import glob
import yaml
# Load the config file into 'c'
c = Configuration(yaml.load(open(os.path.realpath(config_file), 'r')))
#diag = open(os.path.join('/home/data/Projects/CPAC_Regression_Test/2013-08-19-20_v0-3-1/fsl-model/2013-09-03', 'group_runner_diagnostic.txt'), 'wt')
#print >>diag, "Config file: ", c
#print >>diag, ""
#print >>diag, "Output path file: ", output_path_file
#print >>diag, ""
subject_paths = []
for file in glob.glob(os.path.abspath(output_path_file)):
path_list = open(file, 'r').readlines()
subject_paths.extend([s.rstrip('\r\n') for s in path_list])
#print >>diag, "Subject paths list size: "
#print >>diag, len(subject_paths)
#print >>diag, ""
#print >>diag, "First subject path: "
#print >>diag, subject_paths[0]
#print >>diag, ""
set_subject_paths = set(subject_paths)
subject_paths = list(set_subject_paths)
#base_path = os.path.dirname(os.path.commonprefix(subject_paths))
base_path = c.outputDirectory
from collections import defaultdict
analysis_map = defaultdict(list)
analysis_map_gp = defaultdict(list)
for subject_path in subject_paths:
#Remove the base bath offset
rs_path = subject_path.replace(base_path, "", 1)
rs_path = rs_path.lstrip('/')
folders = split_folders(rs_path)
pipeline_id = folders[0]
subject_id = folders[1]
resource_id = folders[2]
scan_id = folders[3]
#if scan_id == '_scan_rest_1_rest':
key = subject_path.replace(subject_id, '*')
analysis_map[(resource_id, key)].append((pipeline_id, subject_id, scan_id, subject_path))
# separate map for group analysis
#if c.mixedScanAnalysis == True:
# key = key.replace(scan_id, '*')
analysis_map_gp[(resource_id, key)].append((pipeline_id, subject_id, scan_id, subject_path))
#print >>diag, ""
#print >>diag, "Analysis_map_gp dictionary size: "
#print >>diag, len(analysis_map_gp)
#print >>diag, ""
#print >>diag, "Derivative list: "
#print >>diag, c.derivativeList
#print >>diag, ""
timing = open(os.path.join(c.outputDirectory, 'group_analysis_timing.txt'), 'wt')
#timing = open(os.path.join('/home/data/Projects/CPAC_Regression_Test/2013-08-19-20_v0-3-1/fsl-model/2013-09-03', 'group_analysis_timing.txt'), 'wt')
sca_roi_runs = 0
sca_roi_time = 0
sca_seed_runs = 0
sca_seed_time = 0
sca_tempreg_runs = 0
sca_tempreg_time = 0
dr_tempreg_runs = 0
dr_tempreg_time = 0
vmhc_z_runs = 0
vmhc_z_time = 0
alff_Z_runs = 0
alff_Z_time = 0
falff_Z_runs = 0
falff_Z_time = 0
reho_Z_runs = 0
reho_Z_time = 0
centrality_outputs_runs = 0
centrality_outputs_time = 0
# Start timing here
gpa_start_time = time.time()
for resource, glob_key in analysis_map.keys():
if resource == 'functional_mni':
wf_start_time = time.time()
if 1 in c.runBASC:
if not c.runOnGrid:
from CPAC.pipeline.cpac_basc_pipeline import prep_basc_workflow
prep_basc_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
if 1 in c.runCWAS:
if not c.runOnGrid:
from CPAC.pipeline.cpac_cwas_pipeline import prep_cwas_workflow
prep_cwas_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
print >>timing, "Group analysis workflow completed for resource: ", resource
print >>timing, "Elapsed run time (minutes): ", ((time.time() - wf_start_time)/60)
print >>timing, ""
for resource, glob_key in analysis_map_gp.keys():
if resource in c.derivativeList:
wf_start_time = time.time()
#print >>diag, "Resource: "
#print >>diag, resource
#print >>diag, ""
#print >>diag, "glob key: "
#print >>diag, glob_key
#print >>diag, ""
#print >>diag, "Analysis map gp entry: "
#print >>diag, analysis_map_gp[(resource,glob_key)]
#print >>diag, ""
if 1 in c.runGroupAnalysis:
#get all the motion parameters across subjects
try:
from CPAC.utils import extract_parameters
extract_parameters.run(c.outputDirectory)
except Exception:
print "Extract parameters script did not run correctly"
if not c.runOnGrid:
from CPAC.pipeline.cpac_group_analysis_pipeline import prep_group_analysis_workflow
#print c, " ", resource, " ", analysis_map_gp[(resource, glob_key)], " ", glob_key
prep_group_analysis_workflow(c, resource, analysis_map_gp[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource, analysis_map_gp[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource, analysis_map_gp[(resource, glob_key)])
print >>timing, "Group analysis workflow completed for resource: ", resource
print >>timing, "Elapsed run time (minutes): ", ((time.time() - wf_start_time)/60)
print >>timing, ""
# This can be implemented more sleekly using a dictionary, have to do this at some point
if resource == 'sca_roi_Z_to_standard_smooth':
sca_roi_runs += 1
sca_roi_time = sca_roi_time + ((time.time() - wf_start_time)/60)
elif resource == 'sca_seed_Z_to_standard_smooth':
sca_seed_runs += 1
sca_seed_time = sca_seed_time + ((time.time() - wf_start_time)/60)
elif resource == 'sca_tempreg_maps_z_files_smooth':
sca_tempreg_runs += 1
sca_tempreg_time = sca_tempreg_time + ((time.time() - wf_start_time)/60)
elif resource == 'dr_tempreg_maps_z_files_smooth':
dr_tempreg_runs += 1
dr_tempreg_time = dr_tempreg_time + ((time.time() - wf_start_time)/60)
elif resource == 'vmhc_z_score_stat_map':
vmhc_z_runs += 1
vmhc_z_time = vmhc_z_time + ((time.time() - wf_start_time)/60)
elif resource == 'alff_Z_to_standard_smooth':
alff_Z_runs += 1
alff_Z_time = alff_Z_time + ((time.time() - wf_start_time)/60)
elif resource == 'falff_Z_to_standard_smooth':
falff_Z_runs += 1
falff_Z_time = falff_Z_time + ((time.time() - wf_start_time)/60)
elif resource == 'reho_Z_to_standard_smooth':
reho_Z_runs += 1
reho_Z_time = reho_Z_time + ((time.time() - wf_start_time)/60)
elif resource == 'centrality_outputs_smoothed':
centrality_outputs_runs += 1
centrality_outputs_time = centrality_outputs_time + ((time.time() - wf_start_time)/60)
print >>timing, "Entire group analysis run complete."
print >>timing, "Elapsed run time (minutes): ", ((time.time() - gpa_start_time)/60)
print >>timing, ""
print >>timing, "sca_roi_Z_to_standard_smooth"
print >>timing, "Number of runs: ", sca_roi_runs
print >>timing, "Total run time (minutes): ", sca_roi_time
print >>timing, ""
print >>timing, "sca_seed_Z_to_standard_smooth"
print >>timing, "Number of runs: ", sca_seed_runs
print >>timing, "Total run time (minutes): ", sca_seed_time
print >>timing, ""
print >>timing, "sca_tempreg_maps_z_files_smooth"
print >>timing, "Number of runs: ", sca_tempreg_runs
print >>timing, "Total run time (minutes): ", sca_tempreg_time
print >>timing, ""
print >>timing, "dr_tempreg_maps_z_files_smooth"
print >>timing, "Number of runs: ", dr_tempreg_runs
print >>timing, "Total run time (minutes): ", dr_tempreg_time
print >>timing, ""
print >>timing, "vmhc_z_score_stat_map"
print >>timing, "Number of runs: ", vmhc_z_runs
print >>timing, "Total run time (minutes): ", vmhc_z_time
print >>timing, ""
print >>timing, "alff_Z_to_standard_smooth"
print >>timing, "Number of runs: ", alff_Z_runs
print >>timing, "Total run time (minutes): ", alff_Z_time
print >>timing, ""
print >>timing, "falff_Z_to_standard_smooth"
print >>timing, "Number of runs: ", falff_Z_runs
print >>timing, "Total run time (minutes): ", falff_Z_time
print >>timing, ""
print >>timing, "reho_Z_to_standard_smooth"
print >>timing, "Number of runs: ", reho_Z_runs
print >>timing, "Total run time (minutes): ", reho_Z_time
print >>timing, ""
print >>timing, "centrality_outputs_smoothed"
print >>timing, "Number of runs: ", centrality_outputs_runs
print >>timing, "Total run time (minutes): ", centrality_outputs_time
print >>timing, ""
timing.close()
def run(config_file, output_path_file):
import re
import os
import glob
import yaml
c = Configuration(yaml.load(open(os.path.realpath(config_file), 'r')))
subject_paths = []
for file in glob.glob(os.path.abspath(output_path_file)):
path_list = open(file, 'r').readlines()
subject_paths.extend([s.rstrip('\r\n') for s in path_list])
set_subject_paths = set(subject_paths)
subject_paths = list(set_subject_paths)
#base_path = os.path.dirname(os.path.commonprefix(subject_paths))
base_path = c.outputDirectory
from collections import defaultdict
analysis_map = defaultdict(list)
analysis_map_gp = defaultdict(list)
for subject_path in subject_paths:
#Remove the base bath offset
rs_path = subject_path.replace(base_path, "", 1)
rs_path = rs_path.lstrip('/')
folders = split_folders(rs_path)
pipeline_id = folders[0]
subject_id = folders[1]
resource_id = folders[2]
scan_id = folders[3]
key = subject_path.replace(subject_id, '*')
analysis_map[(resource_id, key)].append((pipeline_id, subject_id, scan_id, subject_path))
# separate map for group analysis
# if c.mixedScanAnalysis == True:
# key = key.replace(scan_id, '*')
analysis_map_gp[(resource_id, key)].append((pipeline_id, subject_id, scan_id, subject_path))
for resource, glob_key in analysis_map.keys():
if resource == 'functional_mni':
if 1 in c.runBASC:
if not c.runOnGrid:
from CPAC.pipeline.cpac_basc_pipeline import prep_basc_workflow
prep_basc_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
if 1 in c.runCWAS:
if not c.runOnGrid:
from CPAC.pipeline.cpac_cwas_pipeline import prep_cwas_workflow
prep_cwas_workflow(c, analysis_map[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource, analysis_map[(resource, glob_key)])
for resource, glob_key in analysis_map_gp.keys():
if resource in c.derivativeList:
if 1 in c.runGroupAnalysis:
#get all the motion parameters across subjects
try:
from CPAC.utils import extract_parameters
extract_parameters.run(c.outputDirectory)
except Exception:
print "Extract parameters script did not run correctly"
if not c.runOnGrid:
from CPAC.pipeline.cpac_group_analysis_pipeline import prep_group_analysis_workflow
prep_group_analysis_workflow(c, resource, analysis_map_gp[(resource, glob_key)])
else:
if 'sge' in c.resourceManager.lower():
run_sge_jobs(c, config_file, resource, analysis_map_gp[(resource, glob_key)])
elif 'pbs' in c.resourceManager.lower():
run_pbs_jobs(c, config_file, resource, analysis_map_gp[(resource, glob_key)])
