def execCmd(self, cmdList=None, userErrStr='Plugin returned error!'):
"""
This method executes a list of commands (eg. mri_convert) either by scheduling them on a cluster
or just running them locally depending on Chris configuration
"""
if (cmdList is None):
raise ValueError("A list of cmd strings must be passed on")
cmdIds = []
exitCodeFilePaths = [];
chrisRunDir = self.chrisRunDir
envFile = open(chrisRunDir + '/chris.env')
envStr = envFile.read()
envFile.close()
clType = self.clusterType
for cmd, outFileName in itertools.izip(cmdList, self.outputFileNames):
#create stderr, stdout, and exit code log for each cmd execution
ix = outFileName.rfind('.')
cmdId = outFileName[:ix]
cmdIds.append(cmdId)
out = '%s/%s' % (chrisRunDir, cmdId)
exitCodeFilePaths.append(out + '-exitCode.log')
if clType == 'local':
cmd = '%s 2>%s.err >%s.std || (echo $? > %s-exitCode.log)' % (cmd, out, out, out)
else:
#write a dummy cmdId.log file too (this is used later to block the python process until all jobs have finished)
cmd = '%s 2>%s.err >%s.std || (echo $? > %s-exitCode.log);touch %s.log' % (cmd, out, out, out, out)
misc.file_writeOnce(out + '.run', envStr + '\n' + cmd)
os.system('chmod 755 ' + out + '.run')
if clType == 'local':
#jobs are not scheduled on a cluster but run locally
shell = crun.crun()
shell.echo(False)
shell.waitForChild(True)
for cmdId in cmdIds:
shell("/bin/bash " + chrisRunDir + '/' + cmdId + '.run')
else:
#jobs are scheduled on a cluster
remUser = self.remoteUser
remHost = self.remoteHost
remUserId = self.remoteUserIdentity
shell = eval('crun.' + clType.lower() + '(remoteUser=remUser, remoteHost=remHost, remoteUserIdentity=remUserId)')
for cmdId in cmdIds:
shell("/bin/bash " + chrisRunDir + '/' + cmdId + '.run', stdoutflush=True, stderrflush=True)
time.sleep(0.5)
#execute while loop until all cmds have written a file with a name cmdId.log in the chris run dir
allHaveWritten = False
while not allHaveWritten:
dirList = os.listdir(chrisRunDir)
#list of cmd ids found in the chris run dir
foundCmdIds = []
for cmdId in cmdIds:
for name in dirList:
if name == cmdId + '.log':
foundCmdIds.append(cmdId)
#remove found dummy log file
os.remove(chrisRunDir + '/' + name)
break;
if len(foundCmdIds) == len(cmdIds):
allHaveWritten = True
else:
for cmdId in foundCmdIds:
#remove already found ids
cmdIds.remove(cmdId)
time.sleep(10)
for path, outFileName in itertools.izip(exitCodeFilePaths, self.outputFileNames):
# check for existence of chrisRunDir/cmdId-exitCode.log
if os.path.isfile(path):
rfile = open(path)
if rfile.read(1) == "0":
shutil.copyfile(os.path.join(self.tempdir, outFileName), os.path.join(self.options.output, outFileName))
else:
ix = outFileName.rfind('.')
misc.file_writeOnce(os.path.join(chrisRunDir, 'ERROR-user-' + outFileName[:ix] + '.err'), userErrStr + ' ' + outFileName)
rfile.close()
else:
shutil.copyfile(os.path.join(self.tempdir, outFileName), os.path.join(self.options.output, outFileName))
def f_stage4callback(**kwargs):
str_cwd = os.getcwd()
for key, val in kwargs.iteritems():
if key == 'roi': l_roi = val
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
str_cwd = pipeline.workingDir()
log = stage.log()
lst_pval = pipeline.l_pval()
lst_group = pipeline.l_group()
lst_hemi = pipeline.l_hemisphere()
lst_surface = pipeline.l_surface()
lst_statFunc = pipeline.l_statFunc()
lst_roi = pipeline.l_roi()
lst_ctype = ['thickness', 'curv']
lst_leaf = ['pval', 'statFunc', 'pval-statFunc']
shellCluster = crun.crun_hpc_mosix(
remoteUser='******',
remoteHost='rc-majesty',
schedulerStdOutDir='%s-%s' %
(pipeline.schedulerStdOutDir(), '-stage-4'),
schedulerStdErrDir='%s-%s' %
(pipeline.schedulerStdErrDir(), '-stage-4'))
str_curvFunc = ','.join(pipeline.l_curvFunc())
log('Group list: %s\n' % lst_group)
for pipeline._str_group in lst_group:
for pipeline._str_pval in lst_pval:
for pipeline._str_statFunc in lst_statFunc:
for pipeline._str_surface in lst_surface:
for pipeline._str_hemi in lst_hemi:
for ctype in lst_ctype:
if ctype == 'thickness':
str_curvFunc = 'thickness'
else:
l_curv = list(pipeline.l_curvFunc())
log('curvFunc = %s\n' % l_curv)
if any('thickness' in s for s in l_curv):
l_curv.remove('thickness')
str_curvFunc = ','.join(l_curv)
log('Processing %s-%s-%s\n' % \
(pipeline._str_group, pipeline.dirSpecPartial(), ctype))
os.chdir(
'%s/%s/%s' %
(pipeline.dirSpec(), ctype, 'pval-only'))
sortCmd = "/bin/ls -l | sort -n -k 5 | awk '{print $9}'"
l_pval = OSshell(sortCmd)[0].strip().split(
'\n')
l_pval = filter(None, l_pval)
os.chdir('%s/%s/%s' % (pipeline.dirSpec(),
ctype, 'statFunc-only'))
l_statFunc = OSshell(sortCmd)[0].strip().split(
'\n')
l_statFunc = filter(None, l_statFunc)
os.chdir('%s/%s/%s' %
(pipeline.dirSpec(), ctype,
'pval_N_statFunc'))
l_pNs = OSshell(sortCmd)[0].strip().split('\n')
l_pNs = filter(None, l_pNs)
os.chdir('../')
labelCount = 0
rLabelCount = 0
colorMult = 9
misc.file_writeOnce(
'pval-only.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
misc.file_writeOnce(
'statFunc-only.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
misc.file_writeOnce(
'pval_N_statFunc.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
misc.file_writeOnce(
'all.freeview.txt',
'#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF'
% pipeline._str_hemi)
for pvalROI in l_pval:
if pvalROI != 'entire':
intensity = 127 + labelCount * colorMult
if intensity > 255: intensity = 255
misc.file_writeOnce(
'pval-only.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=%d,0,0'
% (pipeline._str_hemi, pvalROI,
intensity),
mode='a')
misc.file_writeOnce(
'pval-only.tcl',
'labl_load %s.%s.label ; labl_set_color %d %d 0 0\n'
% (pipeline._str_hemi, pvalROI,
labelCount, intensity),
mode='a')
misc.file_writeOnce(
'all.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=%d,0,0'
% (pipeline._str_hemi, pvalROI,
intensity),
mode='a')
misc.file_writeOnce(
'all.tcl',
'labl_load %s.%s.label ; labl_set_color %d %d 0 0\n'
% (pipeline._str_hemi, pvalROI,
labelCount, intensity),
mode='a')
labelCount += 1
rLabelCount = labelCount
misc.file_writeOnce(
'pval-only.tcl',
tcl_append(pipeline.namespec(),
'%s-pval-only' % str_curvFunc),
mode='a')
for statFuncROI in l_statFunc:
if statFuncROI != 'entire':
intensity = 127 + (
labelCount -
rLabelCount) * colorMult
if intensity > 255: intensity = 255
misc.file_writeOnce(
'statFunc-only.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,0,%d'
% (pipeline._str_hemi, statFuncROI,
intensity),
mode='a')
misc.file_writeOnce(
'statFunc-only.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 0 %d\n'
% (pipeline._str_hemi, statFuncROI,
labelCount, intensity),
mode='a')
misc.file_writeOnce(
'all.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,0,%d'
% (pipeline._str_hemi, statFuncROI,
intensity),
mode='a')
misc.file_writeOnce(
'all.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 0 %d\n'
% (pipeline._str_hemi, statFuncROI,
labelCount, intensity),
mode='a')
labelCount += 1
rLabelCount = labelCount
misc.file_writeOnce(
'statFunc-only.tcl',
tcl_append(
pipeline.namespec(),
'%s-statFunc-only' % str_curvFunc),
mode='a')
for pNsROI in l_pNs:
if pNsROI != 'entire':
intensity = 127 + (
labelCount -
rLabelCount) * colorMult
if intensity > 255: intensity = 255
misc.file_writeOnce(
'pval_N_statFunc.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,%d,0'
% (pipeline._str_hemi, pNsROI,
intensity),
mode='a')
misc.file_writeOnce(
'pval_N_statFunc.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 %d 0\n'
% (pipeline._str_hemi, pNsROI,
labelCount, intensity),
mode='a')
misc.file_writeOnce(
'all.freeview.txt',
':label=$SUBJECT/label/%s.%s.label:label_color=0,%d,0'
% (pipeline._str_hemi, pNsROI,
intensity),
mode='a')
misc.file_writeOnce(
'all.tcl',
'labl_load %s.%s.label ; labl_set_color %d 0 %d 0\n'
% (pipeline._str_hemi, pNsROI,
labelCount, intensity),
mode='a')
labelCount += 1
misc.file_writeOnce(
'pval_N_statFunc.tcl',
tcl_append(
pipeline.namespec(),
'%s-pval_N_statFunc' % str_curvFunc),
mode='a')
misc.file_writeOnce(
'all.tcl',
tcl_append(pipeline.namespec(),
'%s-all' % str_curvFunc),
mode='a')
# Now run the actual tcl files to generate the tiffs.
log("Current dir = %s\n" % os.getcwd())
str_scriptDir = '/neuro/users/rudolphpienaar/projects/dyslexia-curv-analysis-2/sh'
str_execCmd = 'cd %s; %s/%s -D %s -h %s' % \
(
os.getcwd(),
str_scriptDir,
# pipeline.workingDir(),
"./tksurfer-run.bash",
os.getcwd(),
pipeline._str_hemi,
)
log("Shell command = %s\n" % str_execCmd)
shellCluster.waitForChild(True)
shellCluster.echoStdOut(True)
shellCluster.echoStdErr(True)
shellCluster(str_execCmd)
shellCluster.blockOnChild()
stage.exitCode(0)
return True
def labelScript_process(self, **kwargs):
"""
Write the tcl files to display filtered ROIs
:param kwargs:
:return:
"""
spec = self._d_bootstrapFiltered['%s-filtered' % self._str_group][
self._str_annotation][self._str_hemi][self._str_surface][
self._str_curvFunc].keys()[0]
innerDict = self._d_bootstrapFiltered['%s-filtered' % self._str_group][
self._str_annotation][self._str_hemi][self._str_surface][
self._str_curvFunc][spec]
# print(innerDict)
str_dirSpec = '%s/%s' % (self.dirSpec(), self._str_curvFunc)
os.chdir(str_dirSpec)
str_fileStem = "%s-%s" % (self.namespec("-"), self._str_curvFunc)
str_TCLfileName = '%s.tcl' % (str_fileStem)
str_JSONfileName = '%s.json' % (str_fileStem)
self._log("\n")
str_currentDir = os.getcwd()
l_currentDir = str_currentDir.split('/')
l_workingDir = l_currentDir[-8:-1]
l_workingDir.append(l_currentDir[-1])
index = 0
self._log("Current dir: %s\n" % '/'.join(l_workingDir))
self._log('Creating tcl file: %s...\n' % str_TCLfileName)
with open(str_JSONfileName, 'w') as JSONfile:
json.dump(innerDict, JSONfile, indent=4, sort_keys=True)
self._log('Creating JSON file: %s...\n' % str_JSONfileName)
for key, val in innerDict.iteritems():
if val <= self._f_lowerBoundSoft:
misc.file_writeOnce(str_TCLfileName,
'labl_load %s ; labl_set_color %d 0 0 %d\n' %\
(key, index, 2*int(val)), mode='a')
if val > self._f_lowerBoundSoft and val < self._f_upperBoundSoft:
misc.file_writeOnce(str_TCLfileName,
'labl_load %s ; labl_set_color %d 0 %d 0\n' %\
(key, index, 2*int(val), 2*int(val)), mode='a')
if val >= self._f_upperBoundSoft:
misc.file_writeOnce(str_TCLfileName,
'labl_load %s ; labl_set_color %d %d 0 0\n' %\
(key, index, 2*int(val)), mode='a')
index += 1
misc.file_writeOnce(str_TCLfileName,
self.tcl_append(str_dirSpec, str_fileStem,
self._str_hemi),
mode='a')
str_scriptDir = '/neuro/users/rudolphpienaar/projects/dyslexia-curv-analysis-2/sh'
str_subjectDir = '/neuro/users/rudolphpienaar/projects/dyslexia-curv-analysis-2/results/6-exp-dyslexia-run'
str_execCmd = 'cd %s; %s/%s -S %s -D %s -h %s' % \
(
os.getcwd(),
str_scriptDir,
"./tksurfer-run.bash",
str_subjectDir,
os.getcwd(),
self._str_hemi,
)
self._log("Shell command = %s\n" % str_execCmd)
self._log('Executing tcl file...\n')
OSshell(str_execCmd)
return {"return": "ok"}
def f_stage2callback(**kwargs):
''' Builds list of statFunc tagged hits '''
str_cwd = os.getcwd()
l_roi = []
for key, val in kwargs.iteritems():
if key == 'roi': l_roi = val
if key == 'pipe': pipeline = val
if key == 'obj': stage = val
str_cwd = pipeline.workingDir()
log = stage.log()
lst_statFunc = pipeline.l_statFunc()
lst_group = pipeline.l_group()
lst_pval = pipeline.l_pval()
for roi in l_roi:
log('Processing %s\n' % roi)
for pipeline._str_statFunc in lst_statFunc:
for pipeline._str_pval in lst_pval:
for pipeline._str_surface in pipeline.l_surface():
for pipeline._str_hemi in pipeline.l_hemisphere():
str_summaryFileDir = '%s/%s/dty_analyze-3-cutoffs-%s-%s' % \
(roi, pipeline._str_statFunc, pipeline._str_pval,
args.curvFunc)
# print(str_summaryFileDir)
os.chdir(str_summaryFileDir)
for pipeline._str_group in lst_group:
str_tagFile = 'separate-%s-AreaDensity-%s-%s.txt' % \
(pipeline._str_group, pipeline._str_hemi, pipeline._str_surface)
if os.path.isfile(str_tagFile):
for hit in [
line.strip()
for line in open(str_tagFile, 'r')
]:
hitFile = hit.split()[2]
l_fileSpec = os.path.basename(
hitFile).split('-')
pipeline._str_hemi = l_fileSpec[2]
pipeline._str_func = l_fileSpec[3]
pipeline._str_surface = l_fileSpec[-3]
if pipeline._str_func != 'thickness':
ctype = 'curv'
else:
ctype = 'thickness'
# print(os.getcwd())
# print(l_fileSpec)
# log('hit = %s\n' % hit)
curvMatch = [
s for s in pipeline.l_curvFunc()
if pipeline._str_func == s
]
if len(curvMatch):
log('Logging hit for group %s, %s.%s.%s, p-val theshold %s, ctype %s\n' % \
( pipeline._str_group,
pipeline._str_hemi,
pipeline._str_func,
pipeline._str_surface,
pipeline._str_pval,
ctype))
try:
misc.file_writeOnce('%s/%s/statFunc/%s' % \
(pipeline.dirSpec(), ctype, roi), '%s\n' % hit, mode='a')
except:
log('Ignoring %s-%s\n' %
(pipeline._str_hemi,
pipeline._str_surface))
os.chdir(str_cwd)
return True
