def f_stage2callback(**kwargs):
str_cwd = os.getcwd()
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
for subj in lst_subj:
# find the relevant input files in each <subj> dir
os.chdir(subj)
l_B0 = misc.find(_str_b0)
l_ASL = misc.find(_str_asl)
if not l_B0: error.fatal(pipe_hypothermia, 'noB0')
if not l_ASL: error.fatal(pipe_hypothermia, 'noASL')
_str_B0File = l_B0[0]
_str_ASLFile = l_ASL[0]
_str_outDir = 'outDir'
_str_outFile = 'asl2b0.nii'
misc.mkdir(_str_outDir)
log = stage.log()
log('Scheduling masconorm for "%s: ASL"..\n' % (subj))
str_cmd = 'masconorm.py --input %s/asl2b0.nii.gz --mask %s/b0Brain_mask.nii.gz --outStem %s/asl' % (
_str_outDir,
_str_outDir,
_str_outDir)
#cluster = crun.crun_mosix()
cluster = crun.crun()
cluster.echo(False)
cluster.echoStdOut(False)
cluster.detach(False)
cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True)
if cluster.exitCode():
error.fatal(pipe_hypothermia, 'stageExec', cluster.stderr())
os.chdir(str_cwd)
return True
def f_stage3callback(**kwargs):
str_cwd = os.getcwd()
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
for subj in lst_subj:
# find the relevant input files in each <subj> dir
os.chdir(subj)
l_ADC = misc.find(_str_adc)
if not l_ADC: error.fatal(pipe_hypothermia, 'noADC')
_str_ADCFile = l_ADC[0]
_str_outDir = 'outDir'
misc.mkdir(_str_outDir)
log = stage.log()
log('Scheduling masconorm for "%s: ADC"..\n' % (subj))
str_cmd = 'masconorm.py --input %s --mask %s/b0Brain_mask.nii.gz --outStem %s/adc' % (
_str_ADCFile,
_str_outDir,
_str_outDir)
cluster = crun.crun()
cluster.echo(False)
cluster.echoStdOut(False)
cluster.detach(False)
cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True)
os.chdir(str_cwd)
return True
def f_stage0callback(**kwargs):
str_cwd = os.getcwd()
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
log = stage.log()
lst_annotation = pipeline.l_annotation()
lst_pval = pipeline.l_pval()
lst_group = pipeline.l_group()
lst_hemi = pipeline.l_hemisphere()
lst_surface = pipeline.l_surface()
lst_statFunc = pipeline.l_statFunc()
lst_ctype = ['thickness', 'curv']
lst_leaf = ['pval', 'statFunc', 'pval-only', 'statFunc-only', 'pval_N_statFunc']
for pipeline._str_annotation in lst_annotation:
for pipeline._str_group in lst_group:
for pipeline._str_pval in lst_pval:
for pipeline._str_statFunc in lst_statFunc:
for pipeline._str_surface in lst_surface:
for pipeline._str_hemi in lst_hemi:
for ctype in lst_ctype:
for leaf in lst_leaf:
OSshell('mkdir -p %s/%s/%s' % (
pipeline.dirSpec(),
ctype,
leaf))
stage.exitCode(0)
return True
def f_stage0callback(**kwargs):
str_cwd = os.getcwd()
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
for subj in lst_subj:
# find the relevant input files in each <subj> dir
os.chdir(subj)
l_B0 = misc.find(_str_b0)
if not l_B0: error.fatal(pipe_hypothermia, 'noB0')
_str_B0File = l_B0[0]
_str_outDir = '%s/outDir' % os.getcwd()
_str_outFile = 'b0Brain.nii'
misc.mkdir(_str_outDir)
str_prefixCmd = '( cd %s ; ' % (os.getcwd())
log = stage.log()
log('Scheduling brain extraction for "%s"...\n' % (subj))
str_cmd = 'bet.py --input %s --output %s/%s' % (_str_B0File,
_str_outDir, _str_outFile)
#cluster = crun.crun_mosix(cmdPrefix=str_prefixCmd)
#cluster = crun.crun_mosix()
cluster = crun.crun()
#str_ccmd = cluster.echo(True)
#log(str_ccmd)
cluster.echo(False)
cluster.echoStdOut(False)
cluster.detach(False)
cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True)
if cluster.exitCode():
error.fatal(pipe_hypothermia, 'stageExec', cluster.stderr())
os.chdir(str_cwd)
return True
def f_stage3callback(**kwargs):
''' Sorts intersect/xor of classes '''
str_cwd = os.getcwd()
for key, val in kwargs.iteritems():
if key == 'roi': l_roi = val
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
str_cwd = pipeline.workingDir()
log = stage.log()
lst_pval = pipeline.l_pval()
lst_group = pipeline.l_group()
lst_hemi = pipeline.l_hemisphere()
lst_surface = pipeline.l_surface()
lst_statFunc = pipeline.l_statFunc()
lst_roi = pipeline.l_roi()
lst_ctype = ['thickness', 'curv']
lst_leaf = [
'pval', 'statFunc', 'pval-only', 'statFunc-only', 'pval_N_statFunc'
]
for pipeline._str_group in lst_group:
for pipeline._str_pval in lst_pval:
for pipeline._str_statFunc in lst_statFunc:
for pipeline._str_surface in lst_surface:
for pipeline._str_hemi in lst_hemi:
for ctype in lst_ctype:
log('Processing %s\n' %
pipeline.dirSpecPartial())
os.chdir('%s/%s/%s' %
(pipeline.dirSpec(), ctype, 'pval'))
l_pval = OSshell('ls')[0].strip().split('\n')
os.chdir(
'%s/%s/%s' %
(pipeline.dirSpec(), ctype, 'statFunc'))
l_statFunc = OSshell('ls')[0].strip().split(
'\n')
os.chdir('../')
if l_pval == ['']: sp = set()
else: sp = set(l_pval)
if l_statFunc == ['']: sf = set()
else: sf = set(l_statFunc)
U = sp.union(sf)
I = sp.intersection(sf)
pO = sp - I
fO = sf - I
for roi in list(I):
log('Intersection ROI: %s\n' % roi)
shutil.copyfile('statFunc/%s' % roi,
'pval_N_statFunc/%s' % roi)
for roi in list(pO):
log('p-val only ROI: %s\n' % roi)
shutil.copyfile('pval/%s' % roi,
'pval-only/%s' % roi)
for roi in list(fO):
log('statFunc only ROI: %s\n' % roi)
shutil.copyfile('statFunc/%s' % roi,
'statFunc-only/%s' % roi)
stage.exitCode(0)
return True
def f_stage2callback(**kwargs):
'''
STAGE 2
-------
Run the (new) 2D cloud-based curvature analysis.
'''
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
os.chdir(pipeline.topDir())
log = stage.log()
log('Stage 2 -- running from dir %s\n' % pipeline.topDir())
shellCluster = crun.crun_hpc_mosix(remoteUser='******',
remoteHost='rc-majesty',
schedulerStdOutDir='%s-%s' % (pipeline.schedulerStdOutDir(), '-stage-2'),
schedulerStdErrDir='%s-%s' % (pipeline.schedulerStdErrDir(), '-stage-2'))
for pipeline._str_annotation in pipeline.l_annotation():
for pipeline._str_subsample in pipeline.l_subsample():
'''
Read the ROI.lst file containing the various annotation ROIs,
and for each element, start a separate deviation analysis
process.
'''
str_outDir = "%s%s/groupCurvAnalysis/%s" % \
( pipeline.outDir(),
pipeline._str_subsample,
pipeline._str_annotation )
os.chdir("%s/%s" % (pipeline.topDir(), str_outDir))
with open('ROI.lst') as f: s = f.read()
l_ROI = s.strip().split(' ')
for str_ROI in l_ROI:
for pipeline._str_hemi in pipeline.l_hemisphere():
for pipeline._str_surface in pipeline.l_surface():
str_execCmd = '%s/../../sh/%s -D %s/%s -h %s -s %s %s' % \
(
pipeline.topDir(),
'./ptile_deviation_analysis-raw.bash',
pipeline.topDir(), str_outDir,
pipeline._str_hemi,
pipeline._str_surface,
str_ROI
)
print(os.getcwd())
log("Shell command = %s\n" % str_execCmd)
shellCluster.waitForChild(True)
shellCluster(str_execCmd)
# Sleep for a second so as not to overload the head node
# in tight scheduling loops
time.sleep(1)
shellCluster.blockOnChild()
stage.exitCode(0)
return True
def f_stage1callback(**kwargs):
''' Sorts intersect/xor of classes '''
str_cwd = os.getcwd()
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
log = stage.log()
lst_annotation = pipeline.l_annotation()
lst_subsample = pipeline.l_subsample()
lst_pval = pipeline.l_pval()
lst_group = pipeline.l_group()
lst_hemi = pipeline.l_hemisphere()
lst_surface = pipeline.l_surface()
lst_statFunc = pipeline.l_statFunc()
lst_ctype = pipeline.l_ctype()
c = pipeline.dtree()
for pipeline._str_annotation in lst_annotation:
for pipeline._str_group in lst_group:
for pipeline._str_pval in lst_pval:
for pipeline._str_statFunc in lst_statFunc:
for pipeline._str_surface in lst_surface:
for pipeline._str_hemi in lst_hemi:
for pipeline._str_ctype in lst_ctype:
l_sum = []
d_occurence = {}
for pipeline._str_subsample in lst_subsample:
log('Processing %s %s for %s...' % (pipeline.dirSpecPartial(),
pipeline._str_ctype,
pipeline._str_subsample), lw=170)
#print(pipeline.dirSpecSubsample())
os.chdir(pipeline.dirSpecSubsample())
l_roi = OSshell("cat all.tcl | grep label | awk '{print $2}'")[0].strip().split('\n')
log('[ %02d ]\n' % len(l_roi), syslog=False, rw=20)
#print(l_roi)
c.cdnode(pipeline.dtreeDir())
c.touch('l_roi', l_roi)
c.touch('path', pipeline.dtreeDir())
l_sum += l_roi
c.cdnode('../')
c.touch('l_sum', l_sum)
l_uniq = sorted(set(l_sum))
for roi in l_uniq:
d_occurence[roi] = float(l_sum.count(roi))/float(len(lst_subsample))*100
c.touch('d_occurence', d_occurence)
os.chdir(pipeline.d2path())
# print(os.getcwd())
# print(json.dumps(d_occurence, sort_keys=True, indent=4,
# separators=(',',': ')))
json.dump(d_occurence, open('%s/occurence.txt' % os.getcwd(), "w"),
sort_keys=True,
indent=4,
separators=(',',': ')
)
stage.exitCode(0)
return True
def f_stage4callback(**kwargs):
'''
STAGE 4
-------
Run the ROI tagging.
'''
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
os.chdir(pipeline.topDir())
log = stage.log()
log('Stage 4 -- running from dir %s\n' % pipeline.topDir())
shellCluster = crun.crun_hpc_mosix(
remoteUser='******',
remoteHost='rc-majesty',
schedulerStdOutDir='%s-%s' %
(pipeline.schedulerStdOutDir(), '-stage-4'),
schedulerStdErrDir='%s-%s' %
(pipeline.schedulerStdErrDir(), '-stage-4'))
for pipeline._str_annotation in pipeline.l_annotation():
for pipeline._str_subsample in pipeline.l_subsample():
str_outDir = "%s%s/groupCurvAnalysis/%s" % \
( pipeline.outDir(),
pipeline._str_subsample,
pipeline._str_annotation )
os.chdir("%s/%s" % (pipeline.topDir(), str_outDir))
str_pval = ','.join(pipeline.l_pval())
str_groups = ','.join(pipeline.l_group())
str_surface = ','.join(pipeline.l_surface())
str_statFunc = ','.join(pipeline.l_statFunc())
str_hemi = ','.join(pipeline.l_hemisphere())
str_curv = ','.join(pipeline.l_curvFunc())
str_execCmd = '%s --clobber --workingDir %s/%s --curvFunc %s --pval %s --group %s --surface %s --statFunc %s --hemi %s --stages 01234 --schedulerStdOutDir=%s --schedulerStdErrDir=%s $(cat ./ROI.lst)' % \
(
# pipeline.topDir(), str_outDir,
"/neuro/users/rudolphpienaar/src/devel/roi_tag/roi_tag.py",
pipeline.topDir(), str_outDir,
str_curv,
str_pval,
str_groups,
str_surface,
str_statFunc,
str_hemi,
'/neuro/users/rudolphpienaar/scratch/%s-roi_tag.jobout' % os.getpid(),
'/neuro/users/rudolphpienaar/scratch/%s-roi_tag.joberr' % os.getpid()
)
log("Shell command = %s\n" % str_execCmd)
shellCluster.waitForChild(True)
shellCluster(str_execCmd)
shellCluster.blockOnChild()
stage.exitCode(0)
return True
def f_stage3callback(**kwargs):
'''
STAGE 3
-------
Run the overlap/density analysis in the cloud space.
'''
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
os.chdir(pipeline.topDir())
log = stage.log()
log('Stage 3 -- running from dir %s\n' % pipeline.topDir())
shellCluster = crun.crun_hpc_mosix(
remoteUser='******',
remoteHost='rc-majesty',
schedulerStdOutDir='%s-%s' %
(pipeline.schedulerStdOutDir(), '-stage-3'),
schedulerStdErrDir='%s-%s' %
(pipeline.schedulerStdErrDir(), '-stage-3'))
for pipeline._str_annotation in pipeline.l_annotation():
for pipeline._str_subsample in pipeline.l_subsample():
for pipeline._str_pval in pipeline.l_pval():
str_outDir = "%s%s/groupCurvAnalysis/%s" % \
( pipeline.outDir(),
pipeline._str_subsample,
pipeline._str_annotation )
os.chdir("%s/%s" % (pipeline.topDir(), str_outDir))
str_groups = ','.join(pipeline.l_group())
str_curv = ','.join(pipeline.l_curvFunc())
for pipeline._str_hemi in pipeline.l_hemisphere():
for pipeline._str_surface in pipeline.l_surface():
str_execCmd = '%s/../../sh/%s -D %s/%s -p %s -g %s -h %s -c %s -s %s' % \
(
pipeline.topDir(),
"./dty_analyze-here.bash",
pipeline.topDir(), str_outDir,
pipeline._str_pval,
str_groups,
pipeline._str_hemi,
str_curv,
pipeline._str_surface
)
log("Current dir = %s\n" % os.getcwd())
log("Shell command = %s\n" % str_execCmd)
shellCluster.waitForChild(True)
shellCluster(str_execCmd)
shellCluster.blockOnChild()
stage.exitCode(0)
return True
def f_stage1callback(**kwargs):
'''
STAGE 1
-------
Run the resampler (intelligently) across all the sample experiment
directories.
'''
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
os.chdir(pipeline.topDir())
log = stage.log()
for pipeline._str_annotation in pipeline.l_annotation():
for pipeline._str_subsample in pipeline.l_subsample():
b_parentContainsSamples = False
str_outDir = "%s%s" % \
(
pipeline.outDir(),
pipeline._str_subsample,
)
log('outDir = %s\n' % str_outDir)
# Check if outParentDir contains samples
l_numer = l_filterNumeric(os.listdir(str_outDir))
log('numerical directory count = %d\n' % len(l_numer))
if len(l_numer):
b_parentContainsSamples = True
str_stages = "12"
else:
str_stages = "012"
str_subjList = ' '.join(pipeline._l_subj)
str_execCmd = "%s -o %s -r %s -s %s -a %s %s" % \
(
"/neuro/users/rudolphpienaar/src/devel/roi_tag/rsample.py",
str_outDir,
pipeline._str_threshold,
str_stages,
pipeline._str_annotation,
str_subjList
)
log("Shell command = %s\n" % str_execCmd)
OSshell(str_execCmd)
stage.exitCode(0)
return True
def f_stage0callback(**kwargs):
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
lst_hemi = pipeline.l_hemisphere()
lst_surface = pipeline.l_surface()
lst_curv = pipeline.l_curv()
if int(args.partitions) > _maxPartitions:
error.fatal(hbwm, 'Partition')
for pipeline._str_subj in lst_subj:
for pipeline._str_hemi in lst_hemi:
for pipeline._str_surface in lst_surface:
for pipeline._str_curv in lst_curv:
log = stage.log()
log('Processing %s: %s.%s, %s...\n' % \
(pipeline.subj(), pipeline.hemi(), pipeline.surface(), pipeline.curv()))
str_hostOnlySpec = ''
if len(args.host):
str_hostOnlySpec = "--host %s " % args.host
log('Locking jobs to only run on host -->%s<--\n' %
args.host)
str_debug = ""
if args.b_debug: str_debug = " --debug "
str_queue = ""
if args.queue: str_queue = " --queue %s " % args.queue
str_cmd = "~/src/scripts/hbwm.py -v 10 -s %s %s -r -m %s -f %s -c %s -p %s --cluster %s %s %s %s" % \
(args.stages, str_hostOnlySpec,
pipeline.hemi(), pipeline.surface(), pipeline.curv(), args.partitions,
args.cluster, str_debug, str_queue, pipeline.subj())
print str_cmd
shell = crun.crun()
shell.echo(False)
shell.echoStdOut(False)
shell.detach(False)
shell(str_cmd,
waitForChild=True,
stdoutflush=True,
stderrflush=True)
if shell.exitCode():
error.fatal(hbwm, 'stageExec', shell.stderr())
os.chdir(pipeline.startDir())
return True
def f_fileCheck(**kwargs):
'''
A file exist checker.
'''
stage = None
l_file = []
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'files': l_file = val
log = stage.log()
b_allFilesExist = True
for str_fileName in l_file:
log('Checking on %s...' % str_fileName, lw=85)
b_exists = misc.file_exists(str_fileName)
b_allFilesExist &= b_exists
log('[ %s ]\n' % b_exists, rw=20, syslog=False)
return b_allFilesExist
def f_stage1callback(**kwargs):
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
log = stage.log()
lst_subj = pipeline.l_subj()
os.chdir(pipeline.outDir())
misc.mkdir('groupCurvAnalysis')
os.chdir('groupCurvAnalysis')
misc.mkdir(pipeline.annotationDir())
os.chdir(pipeline.dirSpecAnnot(base='orig'))
for bashFile in glob.glob(r'*.bash'):
shutil.copy(bashFile, pipeline.dirSpecAnnot(base='rsampled'))
for lstFile in glob.glob(r'*.lst'):
shutil.copy(lstFile, pipeline.dirSpecAnnot(base='rsampled'))
str_CMDoriginalROI = "find . -maxdepth 1 -mindepth 1 -type d | grep -v ROItag | awk -F\/ '{print $2}'"
pipeline.l_ROI(OSshell(str_CMDoriginalROI)[0].strip().split('\n'))
lst_ROI = pipeline.l_ROI()
os.chdir(pipeline.dirSpecAnnot(base='rsampled'))
# print(os.getcwd())
total = 0
for pipeline._str_ROI in lst_ROI:
os.chdir(pipeline.dirSpecAnnot(base='rsampled'))
log('Creating ROI dir %s\n' % (pipeline._str_ROI))
misc.mkdir(pipeline._str_ROI)
os.chdir(pipeline._str_ROI)
tagCentroidCMD = 'ls %s/%s/*centroid*txt' % (pipeline.dirSpecAnnot(
base='orig'), pipeline._str_ROI)
pipeline.l_centroidTXT(
OSshell(tagCentroidCMD)[0].strip().split('\n'))
for pipeline._str_centroidTXT in pipeline.l_centroidTXT():
log('\tCopying original sample %s\n' %
os.path.basename(pipeline._str_centroidTXT))
shutil.copy(
pipeline._str_centroidTXT,
os.path.join(pipeline.outDir(), 'groupCurvAnalysis',
pipeline.annotationDir(), pipeline._str_ROI))
total += 1
stage.exitCode(0)
return True
def f_stage2callback(**kwargs):
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
log = stage.log()
lst_subj = pipeline.l_subj()
os.chdir(pipeline.outDir())
str_CMDsubj = "find . -maxdepth 1 -mindepth 1 -type l | awk -F\/ '{print $2}' | sort "
pipeline.l_subjResampled(OSshell(str_CMDsubj)[0].strip().split('\n'))
lst_subjResampled = pipeline.l_subjResampled()
lst_subjResampled.insert(0, 'Subj')
os.chdir(pipeline.dirSpecAnnot(base='rsampled'))
str_CMDoriginalROI = "find . -maxdepth 1 -mindepth 1 -type d | grep -v ROItag | awk -F\/ '{print $2}'"
pipeline.l_ROI(OSshell(str_CMDoriginalROI)[0].strip().split('\n'))
lst_ROI = pipeline.l_ROI()
total = 0
for pipeline._str_ROI in lst_ROI:
os.chdir(
'%s/%s' %
(pipeline.dirSpecAnnot(base='rsampled'), pipeline._str_ROI))
str_CMDcentroid = "ls -1 *centroid*txt"
pipeline.l_centroidTXT(
OSshell(str_CMDcentroid)[0].strip().split('\n'))
log('Switching to ROI %s\n' % pipeline._str_ROI)
for pipeline._str_centroidTXT in pipeline.l_centroidTXT():
log('\tProcessing %s\n' % pipeline._str_centroidTXT)
l_meas = [
line.strip()
for line in open(pipeline._str_centroidTXT, 'r')
]
filt = [x for x in l_meas if x.split()[0] in lst_subjResampled]
FILEfilt = open('%s.filt' % pipeline._str_centroidTXT, 'w')
for line in filt:
FILEfilt.write(' %s\n' % line)
# log('Saved in %s\n' % os.getcwd())
FILEfilt.close()
shutil.move('%s.filt' % pipeline._str_centroidTXT,
pipeline._str_centroidTXT)
total += 1
stage.exitCode(0)
return True
def f_stage0callback(**kwargs):
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
lst_hemi = pipeline.l_hemisphere()
lst_surface = pipeline.l_surface()
lst_curv = pipeline.l_curv()
if int(args.partitions) > _maxPartitions:
error.fatal(hbwm, 'Partition')
for pipeline._str_subj in lst_subj:
for pipeline._str_hemi in lst_hemi:
for pipeline._str_surface in lst_surface:
for pipeline._str_curv in lst_curv:
log = stage.log()
log('Processing %s: %s.%s, %s...\n' % \
(pipeline.subj(), pipeline.hemi(), pipeline.surface(), pipeline.curv()))
str_hostOnlySpec = ''
if len(args.host):
str_hostOnlySpec = "--host %s " % args.host
log('Locking jobs to only run on host -->%s<--\n' % args.host)
str_debug = ""
if args.b_debug: str_debug = " --debug "
str_queue = ""
if args.queue: str_queue = " --queue %s " % args.queue
str_cmd = "~/src/scripts/hbwm.py -v 10 -s %s %s -r -m %s -f %s -c %s -p %s --cluster %s %s %s %s" % \
(args.stages, str_hostOnlySpec,
pipeline.hemi(), pipeline.surface(), pipeline.curv(), args.partitions,
args.cluster, str_debug, str_queue, pipeline.subj())
print str_cmd
shell = crun.crun()
shell.echo(False)
shell.echoStdOut(False)
shell.detach(False)
shell(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True)
if shell.exitCode():
error.fatal(hbwm, 'stageExec', shell.stderr())
os.chdir(pipeline.startDir())
return True
def f_stage0callback(**kwargs):
str_cwd = os.getcwd()
for key, val in kwargs.iteritems():
if key == 'roi': l_roi = val
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
str_cwd = pipeline.workingDir()
log = stage.log()
lst_pval = pipeline.l_pval()
lst_group = pipeline.l_group()
lst_hemi = pipeline.l_hemisphere()
lst_surface = pipeline.l_surface()
lst_statFunc = pipeline.l_statFunc()
lst_roi = pipeline.l_roi()
lst_ctype = pipeline.l_curvFunc()
lst_ctype = ['thickness', 'curv']
lst_leaf = [
'pval', 'statFunc', 'pval-only', 'statFunc-only', 'pval_N_statFunc'
]
os.chdir(str_cwd)
if os.path.isdir(pipeline.outDir()) and not pipeline.clobber():
log('Existing outDir tree found... deleting...\n')
shutil.rmtree(pipeline.outDir())
OSshell('mkdir -p %s' % pipeline.outDir())
os.chdir(pipeline.outDir())
pipeline.outDir(os.getcwd())
os.chdir(str_cwd)
if OSshell.exitCode() != 0: error.fatal(pipeline, 'outDirNotCreate')
for pipeline._str_group in lst_group:
for pipeline._str_pval in lst_pval:
for pipeline._str_statFunc in lst_statFunc:
for pipeline._str_surface in lst_surface:
for pipeline._str_hemi in lst_hemi:
for ctype in lst_ctype:
for leaf in lst_leaf:
OSshell('mkdir -p %s/%s/%s' %
(pipeline.dirSpec(), ctype, leaf))
stage.exitCode(0)
return True
def f_stage0callback(**kwargs):
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
log = stage.log()
lst_subj = pipeline.l_subj()
if os.path.isdir(pipeline.outDir()):
log('Existing outDir tree found... deleting...\n')
shutil.rmtree(pipeline.outDir())
log('Creating output directory...\n')
OSshell('mkdir -p %s' % pipeline.outDir())
print(OSshell.stdout())
if OSshell.exitCode() != 0: error.fatal(pipeline, 'outDirNotCreate')
os.chdir(pipeline.outDir())
pipeline.outDir(os.getcwd())
passcount = 0
failcount = 0
total = 0
for pipeline._str_subj in lst_subj:
f_cutoff = random.random()
log('thresholding %s...' % pipeline._str_subj)
if f_cutoff < pipeline.threshold():
OSshell('ln -s %s/%s .' %
(pipeline.topDir(), pipeline._str_subj))
log('[ passed ]\n', rw=20, syslog=False)
passcount += 1
else:
log('[ failed ]\n', rw=20, syslog=False)
failcount += 1
total += 1
f_passPerc = float(passcount) / float(total) * 100
f_failPerc = float(failcount) / float(total) * 100
log('passed: %d (%5.2f%s)\n' % (passcount, f_passPerc, '%'))
log('failed: %d (%5.2f%s)\n' % (failcount, f_failPerc, '%'))
stage.exitCode(0)
return True
def f_stage0callback(**kwargs):
for key, val in kwargs.iteritems():
if key == 'jobs': jobs = val
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
# Create shell for scheduling/executing on the remote HPC
pipeline.stageShell_createRemoteInstance(args.cluster, stage=stage)
for job in range(0, int(args.jobs)):
log = stage.log()
log('Processing job: %d...\n' % job)
str_cmd = args.cmd
shell = stage.shell()
shell.echo(True)
shell(
str_cmd, waitForChild=shell.waitForChild(),
stdoutflush=True, stderrflush=True
)
if shell.exitCode():
error.fatal(hpc, 'stageExec', shell.stderr())
os.chdir(pipeline.startDir())
return True
def f_stage4callback(**kwargs):
str_cwd = os.getcwd()
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
for subj in lst_subj:
# find the relevant input files in each <subj> dir
_str_outDir = 'outDir'
misc.mkdir(_str_outDir)
os.chdir("%s/outDir" % subj)
log = stage.log()
log('Scheduling MatLAB basac analysis for subject "%s"...\n' % (subj))
str_cmd = 'eval "/chb/pices/arch/x86_64-Linux/bin/matlab -nodesktop -nosplash -nojvm -r \\\"c = basac_drive(\'%s\'); exit\\\"" |tee basac_process.log' % (
os.getcwd()
)
cluster = crun.crun()
cluster.echo(False)
cluster.echoStdOut(False)
cluster.detach(False)
cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True)
os.chdir(str_cwd)
return True
def f_stage0callback(**kwargs):
'''
STAGE 0
-------
Build output directory trees for the analysis.
'''
str_cwd = os.getcwd()
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
log = stage.log()
b_alreadyResampled = False
b_outAnnotDirExist = False
for pipeline._str_annotation in pipeline.l_annotation():
for pipeline._str_subsample in pipeline.l_subsample():
b_parentContainsSamples = False
str_outDir = "%s%s/groupCurvAnalysis/%s" % \
( pipeline.outDir(),
pipeline._str_subsample,
pipeline._str_annotation )
str_outParentDir = "/".join(str_outDir.split('/')[0:-2])
log('Building output tree for "%s", annotation "%s"\n' % \
( str_outParentDir,
pipeline._str_annotation))
b_outAnnotDirExist = os.path.isdir(str_outDir)
if not b_outAnnotDirExist:
misc.mkdir(str_outDir)
# Check if outParentDir contains samples
l_numer = l_filterNumeric(os.listdir(str_outParentDir))
if len(l_numer): b_parentContainsSamples = True
stage.exitCode(0)
return True
def f_stage4callback(**kwargs):
str_cwd = os.getcwd()
for key, val in kwargs.iteritems():
if key == 'roi': l_roi = val
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
str_cwd = pipeline.workingDir()
log = stage.log()
lst_pval = pipeline.l_pval()
lst_group = pipeline.l_group()
lst_hemi = pipeline.l_hemisphere()
lst_surface = pipeline.l_surface()
lst_statFunc = pipeline.l_statFunc()
lst_roi = pipeline.l_roi()
lst_ctype = ['thickness', 'curv']
lst_leaf = ['pval', 'statFunc', 'pval-statFunc']
shellCluster = crun.crun_hpc_mosix(
remoteUser='******',
remoteHost='rc-majesty',
schedulerStdOutDir='%s-%s' %
(pipeline.schedulerStdOutDir(), '-stage-4'),
schedulerStdErrDir='%s-%s' %
(pipeline.schedulerStdErrDir(), '-stage-4'))
str_curvFunc = ','.join(pipeline.l_curvFunc())
log('Group list: %s\n' % lst_group)
for pipeline._str_group in lst_group:
for pipeline._str_pval in lst_pval:
for pipeline._str_statFunc in lst_statFunc:
for pipeline._str_surface in lst_surface:
for pipeline._str_hemi in lst_hemi:
for ctype in lst_ctype:
if ctype == 'thickness':
str_curvFunc = 'thickness'
else:
l_curv = list(pipeline.l_curvFunc())
log('curvFunc = %s\n' % l_curv)
if any('thickness' in s for s in l_curv):
l_curv.remove('thickness')
str_curvFunc = ','.join(l_curv)
log('Processing %s-%s-%s\n' % \
(pipeline._str_group, pipeline.dirSpecPartial(), ctype))
os.chdir(
'%s/%s/%s' %
(pipeline.dirSpec(), ctype, 'pval-only'))
sortCmd = "/bin/ls -l | sort -n -k 5 | awk '{print $9}'"
l_pval = OSshell(sortCmd)[0].strip().split(
'\n')
l_pval = filter(None, l_pval)
os.chdir('%s/%s/%s' % (pipeline.dirSpec(),
ctype, 'statFunc-only'))
l_statFunc = OSshell(sortCmd)[0].strip().split(
'\n')
l_statFunc = filter(None, l_statFunc)
os.chdir('%s/%s/%s' %
(pipeline.dirSpec(), ctype,
'pval_N_statFunc'))
l_pNs = OSshell(sortCmd)[0].strip().split('\n')
l_pNs = filter(None, l_pNs)
os.chdir('../')
labelCount = 0
rLabelCount = 0
colorMult = 9
misc.file_writeOnce(
'pval-only.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
misc.file_writeOnce(
'statFunc-only.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
misc.file_writeOnce(
'pval_N_statFunc.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
misc.file_writeOnce(
'all.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
for pvalROI in l_pval:
if pvalROI != 'entire':
intensity = 127 + labelCount * colorMult
if intensity > 255: intensity = 255
misc.file_writeOnce(
'pval-only.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=%d,0,0'
% (pipeline._str_hemi, pvalROI,
intensity),
mode='a')
misc.file_writeOnce(
'pval-only.tcl',
'labl_load %s.%s.label ; labl_set_color %d %d 0 0\n'
% (pipeline._str_hemi, pvalROI,
labelCount, intensity),
mode='a')
misc.file_writeOnce(
'all.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=%d,0,0'
% (pipeline._str_hemi, pvalROI,
intensity),
mode='a')
misc.file_writeOnce(
'all.tcl',
'labl_load %s.%s.label ; labl_set_color %d %d 0 0\n'
% (pipeline._str_hemi, pvalROI,
labelCount, intensity),
mode='a')
labelCount += 1
rLabelCount = labelCount
misc.file_writeOnce(
'pval-only.tcl',
tcl_append(pipeline.namespec(),
'%s-pval-only' % str_curvFunc),
mode='a')
for statFuncROI in l_statFunc:
if statFuncROI != 'entire':
intensity = 127 + (
labelCount -
rLabelCount) * colorMult
if intensity > 255: intensity = 255
misc.file_writeOnce(
'statFunc-only.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,0,%d'
% (pipeline._str_hemi, statFuncROI,
intensity),
mode='a')
misc.file_writeOnce(
'statFunc-only.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 0 %d\n'
% (pipeline._str_hemi, statFuncROI,
labelCount, intensity),
mode='a')
misc.file_writeOnce(
'all.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,0,%d'
% (pipeline._str_hemi, statFuncROI,
intensity),
mode='a')
misc.file_writeOnce(
'all.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 0 %d\n'
% (pipeline._str_hemi, statFuncROI,
labelCount, intensity),
mode='a')
labelCount += 1
rLabelCount = labelCount
misc.file_writeOnce(
'statFunc-only.tcl',
tcl_append(
pipeline.namespec(),
'%s-statFunc-only' % str_curvFunc),
mode='a')
for pNsROI in l_pNs:
if pNsROI != 'entire':
intensity = 127 + (
labelCount -
rLabelCount) * colorMult
if intensity > 255: intensity = 255
misc.file_writeOnce(
'pval_N_statFunc.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,%d,0'
% (pipeline._str_hemi, pNsROI,
intensity),
mode='a')
misc.file_writeOnce(
'pval_N_statFunc.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 %d 0\n'
% (pipeline._str_hemi, pNsROI,
labelCount, intensity),
mode='a')
misc.file_writeOnce(
'all.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,%d,0'
% (pipeline._str_hemi, pNsROI,
intensity),
mode='a')
misc.file_writeOnce(
'all.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 %d 0\n'
% (pipeline._str_hemi, pNsROI,
labelCount, intensity),
mode='a')
labelCount += 1
misc.file_writeOnce(
'pval_N_statFunc.tcl',
tcl_append(
pipeline.namespec(),
'%s-pval_N_statFunc' % str_curvFunc),
mode='a')
misc.file_writeOnce(
'all.tcl',
tcl_append(pipeline.namespec(),
'%s-all' % str_curvFunc),
mode='a')
# Now run the actual tcl files to generate the tiffs.
log("Current dir = %s\n" % os.getcwd())
str_scriptDir = '/neuro/users/rudolphpienaar/projects/dyslexia-curv-analysis-2/sh'
str_execCmd = 'cd %s; %s/%s -D %s -h %s' % \
(
os.getcwd(),
str_scriptDir,
# pipeline.workingDir(),
"./tksurfer-run.bash",
os.getcwd(),
pipeline._str_hemi,
)
log("Shell command = %s\n" % str_execCmd)
shellCluster.waitForChild(True)
shellCluster.echoStdOut(True)
shellCluster.echoStdErr(True)
shellCluster(str_execCmd)
shellCluster.blockOnChild()
stage.exitCode(0)
return True
def f_stage2callback(**kwargs):
''' Builds list of statFunc tagged hits '''
str_cwd = os.getcwd()
l_roi = []
for key, val in kwargs.iteritems():
if key == 'roi': l_roi = val
if key == 'pipe': pipeline = val
if key == 'obj': stage = val
str_cwd = pipeline.workingDir()
log = stage.log()
lst_statFunc = pipeline.l_statFunc()
lst_group = pipeline.l_group()
lst_pval = pipeline.l_pval()
for roi in l_roi:
log('Processing %s\n' % roi)
for pipeline._str_statFunc in lst_statFunc:
for pipeline._str_pval in lst_pval:
for pipeline._str_surface in pipeline.l_surface():
for pipeline._str_hemi in pipeline.l_hemisphere():
str_summaryFileDir = '%s/%s/dty_analyze-3-cutoffs-%s-%s' % \
(roi, pipeline._str_statFunc, pipeline._str_pval,
args.curvFunc)
# print(str_summaryFileDir)
os.chdir(str_summaryFileDir)
for pipeline._str_group in lst_group:
str_tagFile = 'separate-%s-AreaDensity-%s-%s.txt' % \
(pipeline._str_group, pipeline._str_hemi, pipeline._str_surface)
if os.path.isfile(str_tagFile):
for hit in [
line.strip()
for line in open(str_tagFile, 'r')
]:
hitFile = hit.split()[2]
l_fileSpec = os.path.basename(
hitFile).split('-')
pipeline._str_hemi = l_fileSpec[2]
pipeline._str_func = l_fileSpec[3]
pipeline._str_surface = l_fileSpec[-3]
if pipeline._str_func != 'thickness':
ctype = 'curv'
else:
ctype = 'thickness'
# print(os.getcwd())
# print(l_fileSpec)
# log('hit = %s\n' % hit)
curvMatch = [
s for s in pipeline.l_curvFunc()
if pipeline._str_func == s
]
if len(curvMatch):
log('Logging hit for group %s, %s.%s.%s, p-val theshold %s, ctype %s\n' % \
( pipeline._str_group,
pipeline._str_hemi,
pipeline._str_func,
pipeline._str_surface,
pipeline._str_pval,
ctype))
try:
misc.file_writeOnce('%s/%s/statFunc/%s' % \
(pipeline.dirSpec(), ctype, roi), '%s\n' % hit, mode='a')
except:
log('Ignoring %s-%s\n' %
(pipeline._str_hemi,
pipeline._str_surface))
os.chdir(str_cwd)
return True
