def f_stage2callback(**kwargs):
'''
STAGE 2
-------
Run the (new) 2D cloud-based curvature analysis.
'''
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
os.chdir(pipeline.topDir())
log = stage.log()
log('Stage 2 -- running from dir %s\n' % pipeline.topDir())
shellCluster = crun.crun_hpc_mosix(remoteUser='******',
remoteHost='rc-majesty',
schedulerStdOutDir='%s-%s' % (pipeline.schedulerStdOutDir(), '-stage-2'),
schedulerStdErrDir='%s-%s' % (pipeline.schedulerStdErrDir(), '-stage-2'))
for pipeline._str_annotation in pipeline.l_annotation():
for pipeline._str_subsample in pipeline.l_subsample():
'''
Read the ROI.lst file containing the various annotation ROIs,
and for each element, start a separate deviation analysis
process.
'''
str_outDir = "%s%s/groupCurvAnalysis/%s" % \
( pipeline.outDir(),
pipeline._str_subsample,
pipeline._str_annotation )
os.chdir("%s/%s" % (pipeline.topDir(), str_outDir))
with open('ROI.lst') as f: s = f.read()
l_ROI = s.strip().split(' ')
for str_ROI in l_ROI:
for pipeline._str_hemi in pipeline.l_hemisphere():
for pipeline._str_surface in pipeline.l_surface():
str_execCmd = '%s/../../sh/%s -D %s/%s -h %s -s %s %s' % \
(
pipeline.topDir(),
'./ptile_deviation_analysis-raw.bash',
pipeline.topDir(), str_outDir,
pipeline._str_hemi,
pipeline._str_surface,
str_ROI
)
print(os.getcwd())
log("Shell command = %s\n" % str_execCmd)
shellCluster.waitForChild(True)
shellCluster(str_execCmd)
# Sleep for a second so as not to overload the head node
# in tight scheduling loops
time.sleep(1)
shellCluster.blockOnChild()
stage.exitCode(0)
return True
def f_stage4callback(**kwargs):
'''
STAGE 4
-------
Run the ROI tagging.
'''
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
os.chdir(pipeline.topDir())
log = stage.log()
log('Stage 4 -- running from dir %s\n' % pipeline.topDir())
shellCluster = crun.crun_hpc_mosix(
remoteUser='******',
remoteHost='rc-majesty',
schedulerStdOutDir='%s-%s' %
(pipeline.schedulerStdOutDir(), '-stage-4'),
schedulerStdErrDir='%s-%s' %
(pipeline.schedulerStdErrDir(), '-stage-4'))
for pipeline._str_annotation in pipeline.l_annotation():
for pipeline._str_subsample in pipeline.l_subsample():
str_outDir = "%s%s/groupCurvAnalysis/%s" % \
( pipeline.outDir(),
pipeline._str_subsample,
pipeline._str_annotation )
os.chdir("%s/%s" % (pipeline.topDir(), str_outDir))
str_pval = ','.join(pipeline.l_pval())
str_groups = ','.join(pipeline.l_group())
str_surface = ','.join(pipeline.l_surface())
str_statFunc = ','.join(pipeline.l_statFunc())
str_hemi = ','.join(pipeline.l_hemisphere())
str_curv = ','.join(pipeline.l_curvFunc())
str_execCmd = '%s --clobber --workingDir %s/%s --curvFunc %s --pval %s --group %s --surface %s --statFunc %s --hemi %s --stages 01234 --schedulerStdOutDir=%s --schedulerStdErrDir=%s $(cat ./ROI.lst)' % \
(
# pipeline.topDir(), str_outDir,
"/neuro/users/rudolphpienaar/src/devel/roi_tag/roi_tag.py",
pipeline.topDir(), str_outDir,
str_curv,
str_pval,
str_groups,
str_surface,
str_statFunc,
str_hemi,
'/neuro/users/rudolphpienaar/scratch/%s-roi_tag.jobout' % os.getpid(),
'/neuro/users/rudolphpienaar/scratch/%s-roi_tag.joberr' % os.getpid()
)
log("Shell command = %s\n" % str_execCmd)
shellCluster.waitForChild(True)
shellCluster(str_execCmd)
shellCluster.blockOnChild()
stage.exitCode(0)
return True
def stageShell_createRemoteInstance(self, astr_remoteHPC, **kwargs):
'''
Returns a crun object in the passed stage object that
functions as a shell on the remote HPC.
'''
for key, val in kwargs.iteritems():
if key == 'stage': stage = val
for case in misc.switch(astr_remoteHPC):
if case('PICES'):
stage.shell(crun.crun_hpc_mosix(
remoteUser="******",
remoteHost="rc-majesty.tch.harvard.edu")
)
stage.shell().emailUser('*****@*****.**')
b_jobDetach = True
b_disassocaite = True
b_waitForChild = False
break
if case('launchpad'):
stage.shell(crun.crun_hpc_launchpad(
remoteUser="******",
remoteHost="fnndsc:7774")
)
b_jobDetach = False
b_disassocaite = False
b_waitForChild = True
break
if case('erisone'):
stage.shell(crun.crun_hpc_lsf(
remoteUser="******",
remoteHost="fnndsc:7773")
)
stage.shell().scheduleHostOnly(
"cmu058 cmu059 cmu061 cmu066 cmu067 cmu071 cmu073 cmu075 cmu077 cmu079 cmu081 cmu087 cmu090 cmu093 cmu094 cmu095 cmu096 cmu102 cmu106 cmu107 cmu108 cmu109 cmu111 cmu112 cmu114 cmu121 cmu123 cmu126 cmu149 cmu154 cmu156 cmu157 "
)
b_jobDetach = False
b_disassocaite = False
b_waitForChild = True
break
if case():
error.fatal(self, 'noClusterSpec')
shell = stage.shell()
shell.emailWhenDone(True)
if args.b_debug:
shell.echo(True)
shell.echoStdOut(True)
else:
shell.echo(False)
shell.echoStdOut(False)
shell.detach(b_jobDetach)
shell.disassociate(b_disassocaite)
shell.waitForChild(b_waitForChild)
def f_stage3callback(**kwargs):
'''
STAGE 3
-------
Run the overlap/density analysis in the cloud space.
'''
for key, val in kwargs.iteritems():
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
os.chdir(pipeline.topDir())
log = stage.log()
log('Stage 3 -- running from dir %s\n' % pipeline.topDir())
shellCluster = crun.crun_hpc_mosix(
remoteUser='******',
remoteHost='rc-majesty',
schedulerStdOutDir='%s-%s' %
(pipeline.schedulerStdOutDir(), '-stage-3'),
schedulerStdErrDir='%s-%s' %
(pipeline.schedulerStdErrDir(), '-stage-3'))
for pipeline._str_annotation in pipeline.l_annotation():
for pipeline._str_subsample in pipeline.l_subsample():
for pipeline._str_pval in pipeline.l_pval():
str_outDir = "%s%s/groupCurvAnalysis/%s" % \
( pipeline.outDir(),
pipeline._str_subsample,
pipeline._str_annotation )
os.chdir("%s/%s" % (pipeline.topDir(), str_outDir))
str_groups = ','.join(pipeline.l_group())
str_curv = ','.join(pipeline.l_curvFunc())
for pipeline._str_hemi in pipeline.l_hemisphere():
for pipeline._str_surface in pipeline.l_surface():
str_execCmd = '%s/../../sh/%s -D %s/%s -p %s -g %s -h %s -c %s -s %s' % \
(
pipeline.topDir(),
"./dty_analyze-here.bash",
pipeline.topDir(), str_outDir,
pipeline._str_pval,
str_groups,
pipeline._str_hemi,
str_curv,
pipeline._str_surface
)
log("Current dir = %s\n" % os.getcwd())
log("Shell command = %s\n" % str_execCmd)
shellCluster.waitForChild(True)
shellCluster(str_execCmd)
shellCluster.blockOnChild()
stage.exitCode(0)
return True
def f_stage4callback(**kwargs):
str_cwd = os.getcwd()
for key, val in kwargs.iteritems():
if key == 'roi': l_roi = val
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
str_cwd = pipeline.workingDir()
log = stage.log()
lst_pval = pipeline.l_pval()
lst_group = pipeline.l_group()
lst_hemi = pipeline.l_hemisphere()
lst_surface = pipeline.l_surface()
lst_statFunc = pipeline.l_statFunc()
lst_roi = pipeline.l_roi()
lst_ctype = ['thickness', 'curv']
lst_leaf = ['pval', 'statFunc', 'pval-statFunc']
shellCluster = crun.crun_hpc_mosix(
remoteUser='******',
remoteHost='rc-majesty',
schedulerStdOutDir='%s-%s' %
(pipeline.schedulerStdOutDir(), '-stage-4'),
schedulerStdErrDir='%s-%s' %
(pipeline.schedulerStdErrDir(), '-stage-4'))
str_curvFunc = ','.join(pipeline.l_curvFunc())
log('Group list: %s\n' % lst_group)
for pipeline._str_group in lst_group:
for pipeline._str_pval in lst_pval:
for pipeline._str_statFunc in lst_statFunc:
for pipeline._str_surface in lst_surface:
for pipeline._str_hemi in lst_hemi:
for ctype in lst_ctype:
if ctype == 'thickness':
str_curvFunc = 'thickness'
else:
l_curv = list(pipeline.l_curvFunc())
log('curvFunc = %s\n' % l_curv)
if any('thickness' in s for s in l_curv):
l_curv.remove('thickness')
str_curvFunc = ','.join(l_curv)
log('Processing %s-%s-%s\n' % \
(pipeline._str_group, pipeline.dirSpecPartial(), ctype))
os.chdir(
'%s/%s/%s' %
(pipeline.dirSpec(), ctype, 'pval-only'))
sortCmd = "/bin/ls -l | sort -n -k 5 | awk '{print $9}'"
l_pval = OSshell(sortCmd)[0].strip().split(
'\n')
l_pval = filter(None, l_pval)
os.chdir('%s/%s/%s' % (pipeline.dirSpec(),
ctype, 'statFunc-only'))
l_statFunc = OSshell(sortCmd)[0].strip().split(
'\n')
l_statFunc = filter(None, l_statFunc)
os.chdir('%s/%s/%s' %
(pipeline.dirSpec(), ctype,
'pval_N_statFunc'))
l_pNs = OSshell(sortCmd)[0].strip().split('\n')
l_pNs = filter(None, l_pNs)
os.chdir('../')
labelCount = 0
rLabelCount = 0
colorMult = 9
misc.file_writeOnce(
'pval-only.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
misc.file_writeOnce(
'statFunc-only.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
misc.file_writeOnce(
'pval_N_statFunc.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
misc.file_writeOnce(
'all.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
for pvalROI in l_pval:
if pvalROI != 'entire':
intensity = 127 + labelCount * colorMult
if intensity > 255: intensity = 255
misc.file_writeOnce(
'pval-only.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=%d,0,0'
% (pipeline._str_hemi, pvalROI,
intensity),
mode='a')
misc.file_writeOnce(
'pval-only.tcl',
'labl_load %s.%s.label ; labl_set_color %d %d 0 0\n'
% (pipeline._str_hemi, pvalROI,
labelCount, intensity),
mode='a')
misc.file_writeOnce(
'all.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=%d,0,0'
% (pipeline._str_hemi, pvalROI,
intensity),
mode='a')
misc.file_writeOnce(
'all.tcl',
'labl_load %s.%s.label ; labl_set_color %d %d 0 0\n'
% (pipeline._str_hemi, pvalROI,
labelCount, intensity),
mode='a')
labelCount += 1
rLabelCount = labelCount
misc.file_writeOnce(
'pval-only.tcl',
tcl_append(pipeline.namespec(),
'%s-pval-only' % str_curvFunc),
mode='a')
for statFuncROI in l_statFunc:
if statFuncROI != 'entire':
intensity = 127 + (
labelCount -
rLabelCount) * colorMult
if intensity > 255: intensity = 255
misc.file_writeOnce(
'statFunc-only.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,0,%d'
% (pipeline._str_hemi, statFuncROI,
intensity),
mode='a')
misc.file_writeOnce(
'statFunc-only.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 0 %d\n'
% (pipeline._str_hemi, statFuncROI,
labelCount, intensity),
mode='a')
misc.file_writeOnce(
'all.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,0,%d'
% (pipeline._str_hemi, statFuncROI,
intensity),
mode='a')
misc.file_writeOnce(
'all.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 0 %d\n'
% (pipeline._str_hemi, statFuncROI,
labelCount, intensity),
mode='a')
labelCount += 1
rLabelCount = labelCount
misc.file_writeOnce(
'statFunc-only.tcl',
tcl_append(
pipeline.namespec(),
'%s-statFunc-only' % str_curvFunc),
mode='a')
for pNsROI in l_pNs:
if pNsROI != 'entire':
intensity = 127 + (
labelCount -
rLabelCount) * colorMult
if intensity > 255: intensity = 255
misc.file_writeOnce(
'pval_N_statFunc.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,%d,0'
% (pipeline._str_hemi, pNsROI,
intensity),
mode='a')
misc.file_writeOnce(
'pval_N_statFunc.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 %d 0\n'
% (pipeline._str_hemi, pNsROI,
labelCount, intensity),
mode='a')
misc.file_writeOnce(
'all.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,%d,0'
% (pipeline._str_hemi, pNsROI,
intensity),
mode='a')
misc.file_writeOnce(
'all.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 %d 0\n'
% (pipeline._str_hemi, pNsROI,
labelCount, intensity),
mode='a')
labelCount += 1
misc.file_writeOnce(
'pval_N_statFunc.tcl',
tcl_append(
pipeline.namespec(),
'%s-pval_N_statFunc' % str_curvFunc),
mode='a')
misc.file_writeOnce(
'all.tcl',
tcl_append(pipeline.namespec(),
'%s-all' % str_curvFunc),
mode='a')
# Now run the actual tcl files to generate the tiffs.
log("Current dir = %s\n" % os.getcwd())
str_scriptDir = '/neuro/users/rudolphpienaar/projects/dyslexia-curv-analysis-2/sh'
str_execCmd = 'cd %s; %s/%s -D %s -h %s' % \
(
os.getcwd(),
str_scriptDir,
# pipeline.workingDir(),
"./tksurfer-run.bash",
os.getcwd(),
pipeline._str_hemi,
)
log("Shell command = %s\n" % str_execCmd)
shellCluster.waitForChild(True)
shellCluster.echoStdOut(True)
shellCluster.echoStdErr(True)
shellCluster(str_execCmd)
shellCluster.blockOnChild()
stage.exitCode(0)
return True