def execCmd(self, cmdList=None, userErrStr='Plugin returned error!'): """ This method executes a list of commands (eg. mri_convert) either by scheduling them on a cluster or just running them locally depending on Chris configuration """ if (cmdList is None): raise ValueError("A list of cmd strings must be passed on") cmdIds = [] exitCodeFilePaths = []; chrisRunDir = self.chrisRunDir envFile = open(chrisRunDir + '/chris.env') envStr = envFile.read() envFile.close() clType = self.clusterType for cmd, outFileName in itertools.izip(cmdList, self.outputFileNames): #create stderr, stdout, and exit code log for each cmd execution ix = outFileName.rfind('.') cmdId = outFileName[:ix] cmdIds.append(cmdId) out = '%s/%s' % (chrisRunDir, cmdId) exitCodeFilePaths.append(out + '-exitCode.log') if clType == 'local': cmd = '%s 2>%s.err >%s.std || (echo $? > %s-exitCode.log)' % (cmd, out, out, out) else: #write a dummy cmdId.log file too (this is used later to block the python process until all jobs have finished) cmd = '%s 2>%s.err >%s.std || (echo $? > %s-exitCode.log);touch %s.log' % (cmd, out, out, out, out) misc.file_writeOnce(out + '.run', envStr + '\n' + cmd) os.system('chmod 755 ' + out + '.run') if clType == 'local': #jobs are not scheduled on a cluster but run locally shell = crun.crun() shell.echo(False) shell.waitForChild(True) for cmdId in cmdIds: shell("/bin/bash " + chrisRunDir + '/' + cmdId + '.run') else: #jobs are scheduled on a cluster remUser = self.remoteUser remHost = self.remoteHost remUserId = self.remoteUserIdentity shell = eval('crun.' + clType.lower() + '(remoteUser=remUser, remoteHost=remHost, remoteUserIdentity=remUserId)') for cmdId in cmdIds: shell("/bin/bash " + chrisRunDir + '/' + cmdId + '.run', stdoutflush=True, stderrflush=True) time.sleep(0.5) #execute while loop until all cmds have written a file with a name cmdId.log in the chris run dir allHaveWritten = False while not allHaveWritten: dirList = os.listdir(chrisRunDir) #list of cmd ids found in the chris run dir foundCmdIds = [] for cmdId in cmdIds: for name in dirList: if name == cmdId + '.log': foundCmdIds.append(cmdId) #remove found dummy log file os.remove(chrisRunDir + '/' + name) break; if len(foundCmdIds) == len(cmdIds): allHaveWritten = True else: for cmdId in foundCmdIds: #remove already found ids cmdIds.remove(cmdId) time.sleep(10) for path, outFileName in itertools.izip(exitCodeFilePaths, self.outputFileNames): # check for existence of chrisRunDir/cmdId-exitCode.log if os.path.isfile(path): rfile = open(path) if rfile.read(1) == "0": shutil.copyfile(os.path.join(self.tempdir, outFileName), os.path.join(self.options.output, outFileName)) else: ix = outFileName.rfind('.') misc.file_writeOnce(os.path.join(chrisRunDir, 'ERROR-user-' + outFileName[:ix] + '.err'), userErrStr + ' ' + outFileName) rfile.close() else: shutil.copyfile(os.path.join(self.tempdir, outFileName), os.path.join(self.options.output, outFileName))
def f_stage4callback(**kwargs): str_cwd = os.getcwd() for key, val in kwargs.iteritems(): if key == 'roi': l_roi = val if key == 'obj': stage = val if key == 'pipe': pipeline = val str_cwd = pipeline.workingDir() log = stage.log() lst_pval = pipeline.l_pval() lst_group = pipeline.l_group() lst_hemi = pipeline.l_hemisphere() lst_surface = pipeline.l_surface() lst_statFunc = pipeline.l_statFunc() lst_roi = pipeline.l_roi() lst_ctype = ['thickness', 'curv'] lst_leaf = ['pval', 'statFunc', 'pval-statFunc'] shellCluster = crun.crun_hpc_mosix( remoteUser='******', remoteHost='rc-majesty', schedulerStdOutDir='%s-%s' % (pipeline.schedulerStdOutDir(), '-stage-4'), schedulerStdErrDir='%s-%s' % (pipeline.schedulerStdErrDir(), '-stage-4')) str_curvFunc = ','.join(pipeline.l_curvFunc()) log('Group list: %s\n' % lst_group) for pipeline._str_group in lst_group: for pipeline._str_pval in lst_pval: for pipeline._str_statFunc in lst_statFunc: for pipeline._str_surface in lst_surface: for pipeline._str_hemi in lst_hemi: for ctype in lst_ctype: if ctype == 'thickness': str_curvFunc = 'thickness' else: l_curv = list(pipeline.l_curvFunc()) log('curvFunc = %s\n' % l_curv) if any('thickness' in s for s in l_curv): l_curv.remove('thickness') str_curvFunc = ','.join(l_curv) log('Processing %s-%s-%s\n' % \ (pipeline._str_group, pipeline.dirSpecPartial(), ctype)) os.chdir( '%s/%s/%s' % (pipeline.dirSpec(), ctype, 'pval-only')) sortCmd = "/bin/ls -l | sort -n -k 5 | awk '{print $9}'" l_pval = OSshell(sortCmd)[0].strip().split( '\n') l_pval = filter(None, l_pval) os.chdir('%s/%s/%s' % (pipeline.dirSpec(), ctype, 'statFunc-only')) l_statFunc = OSshell(sortCmd)[0].strip().split( '\n') l_statFunc = filter(None, l_statFunc) os.chdir('%s/%s/%s' % (pipeline.dirSpec(), ctype, 'pval_N_statFunc')) l_pNs = OSshell(sortCmd)[0].strip().split('\n') l_pNs = filter(None, l_pNs) os.chdir('../') labelCount = 0 rLabelCount = 0 colorMult = 9 misc.file_writeOnce( 'pval-only.freeview.txt', '#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF' % pipeline._str_hemi) misc.file_writeOnce( 'statFunc-only.freeview.txt', '#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF' % pipeline._str_hemi) misc.file_writeOnce( 'pval_N_statFunc.freeview.txt', '#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF' % pipeline._str_hemi) misc.file_writeOnce( 'all.freeview.txt', '#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF' % pipeline._str_hemi) for pvalROI in l_pval: if pvalROI != 'entire': intensity = 127 + labelCount * colorMult if intensity > 255: intensity = 255 misc.file_writeOnce( 'pval-only.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=%d,0,0' % (pipeline._str_hemi, pvalROI, intensity), mode='a') misc.file_writeOnce( 'pval-only.tcl', 'labl_load %s.%s.label ; labl_set_color %d %d 0 0\n' % (pipeline._str_hemi, pvalROI, labelCount, intensity), mode='a') misc.file_writeOnce( 'all.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=%d,0,0' % (pipeline._str_hemi, pvalROI, intensity), mode='a') misc.file_writeOnce( 'all.tcl', 'labl_load %s.%s.label ; labl_set_color %d %d 0 0\n' % (pipeline._str_hemi, pvalROI, labelCount, intensity), mode='a') labelCount += 1 rLabelCount = labelCount misc.file_writeOnce( 'pval-only.tcl', tcl_append(pipeline.namespec(), '%s-pval-only' % str_curvFunc), mode='a') for statFuncROI in l_statFunc: if statFuncROI != 'entire': intensity = 127 + ( labelCount - rLabelCount) * colorMult if intensity > 255: intensity = 255 misc.file_writeOnce( 'statFunc-only.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=0,0,%d' % (pipeline._str_hemi, statFuncROI, intensity), mode='a') misc.file_writeOnce( 'statFunc-only.tcl', 'labl_load %s.%s.label ; labl_set_color %d 0 0 %d\n' % (pipeline._str_hemi, statFuncROI, labelCount, intensity), mode='a') misc.file_writeOnce( 'all.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=0,0,%d' % (pipeline._str_hemi, statFuncROI, intensity), mode='a') misc.file_writeOnce( 'all.tcl', 'labl_load %s.%s.label ; labl_set_color %d 0 0 %d\n' % (pipeline._str_hemi, statFuncROI, labelCount, intensity), mode='a') labelCount += 1 rLabelCount = labelCount misc.file_writeOnce( 'statFunc-only.tcl', tcl_append( pipeline.namespec(), '%s-statFunc-only' % str_curvFunc), mode='a') for pNsROI in l_pNs: if pNsROI != 'entire': intensity = 127 + ( labelCount - rLabelCount) * colorMult if intensity > 255: intensity = 255 misc.file_writeOnce( 'pval_N_statFunc.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=0,%d,0' % (pipeline._str_hemi, pNsROI, intensity), mode='a') misc.file_writeOnce( 'pval_N_statFunc.tcl', 'labl_load %s.%s.label ; labl_set_color %d 0 %d 0\n' % (pipeline._str_hemi, pNsROI, labelCount, intensity), mode='a') misc.file_writeOnce( 'all.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=0,%d,0' % (pipeline._str_hemi, pNsROI, intensity), mode='a') misc.file_writeOnce( 'all.tcl', 'labl_load %s.%s.label ; labl_set_color %d 0 %d 0\n' % (pipeline._str_hemi, pNsROI, labelCount, intensity), mode='a') labelCount += 1 misc.file_writeOnce( 'pval_N_statFunc.tcl', tcl_append( pipeline.namespec(), '%s-pval_N_statFunc' % str_curvFunc), mode='a') misc.file_writeOnce( 'all.tcl', tcl_append(pipeline.namespec(), '%s-all' % str_curvFunc), mode='a') # Now run the actual tcl files to generate the tiffs. log("Current dir = %s\n" % os.getcwd()) str_scriptDir = '/neuro/users/rudolphpienaar/projects/dyslexia-curv-analysis-2/sh' str_execCmd = 'cd %s; %s/%s -D %s -h %s' % \ ( os.getcwd(), str_scriptDir, # pipeline.workingDir(), "./tksurfer-run.bash", os.getcwd(), pipeline._str_hemi, ) log("Shell command = %s\n" % str_execCmd) shellCluster.waitForChild(True) shellCluster.echoStdOut(True) shellCluster.echoStdErr(True) shellCluster(str_execCmd) shellCluster.blockOnChild() stage.exitCode(0) return True
def labelScript_process(self, **kwargs): """ Write the tcl files to display filtered ROIs :param kwargs: :return: """ spec = self._d_bootstrapFiltered['%s-filtered' % self._str_group][ self._str_annotation][self._str_hemi][self._str_surface][ self._str_curvFunc].keys()[0] innerDict = self._d_bootstrapFiltered['%s-filtered' % self._str_group][ self._str_annotation][self._str_hemi][self._str_surface][ self._str_curvFunc][spec] # print(innerDict) str_dirSpec = '%s/%s' % (self.dirSpec(), self._str_curvFunc) os.chdir(str_dirSpec) str_fileStem = "%s-%s" % (self.namespec("-"), self._str_curvFunc) str_TCLfileName = '%s.tcl' % (str_fileStem) str_JSONfileName = '%s.json' % (str_fileStem) self._log("\n") str_currentDir = os.getcwd() l_currentDir = str_currentDir.split('/') l_workingDir = l_currentDir[-8:-1] l_workingDir.append(l_currentDir[-1]) index = 0 self._log("Current dir: %s\n" % '/'.join(l_workingDir)) self._log('Creating tcl file: %s...\n' % str_TCLfileName) with open(str_JSONfileName, 'w') as JSONfile: json.dump(innerDict, JSONfile, indent=4, sort_keys=True) self._log('Creating JSON file: %s...\n' % str_JSONfileName) for key, val in innerDict.iteritems(): if val <= self._f_lowerBoundSoft: misc.file_writeOnce(str_TCLfileName, 'labl_load %s ; labl_set_color %d 0 0 %d\n' %\ (key, index, 2*int(val)), mode='a') if val > self._f_lowerBoundSoft and val < self._f_upperBoundSoft: misc.file_writeOnce(str_TCLfileName, 'labl_load %s ; labl_set_color %d 0 %d 0\n' %\ (key, index, 2*int(val), 2*int(val)), mode='a') if val >= self._f_upperBoundSoft: misc.file_writeOnce(str_TCLfileName, 'labl_load %s ; labl_set_color %d %d 0 0\n' %\ (key, index, 2*int(val)), mode='a') index += 1 misc.file_writeOnce(str_TCLfileName, self.tcl_append(str_dirSpec, str_fileStem, self._str_hemi), mode='a') str_scriptDir = '/neuro/users/rudolphpienaar/projects/dyslexia-curv-analysis-2/sh' str_subjectDir = '/neuro/users/rudolphpienaar/projects/dyslexia-curv-analysis-2/results/6-exp-dyslexia-run' str_execCmd = 'cd %s; %s/%s -S %s -D %s -h %s' % \ ( os.getcwd(), str_scriptDir, "./tksurfer-run.bash", str_subjectDir, os.getcwd(), self._str_hemi, ) self._log("Shell command = %s\n" % str_execCmd) self._log('Executing tcl file...\n') OSshell(str_execCmd) return {"return": "ok"}
def f_stage2callback(**kwargs): ''' Builds list of statFunc tagged hits ''' str_cwd = os.getcwd() l_roi = [] for key, val in kwargs.iteritems(): if key == 'roi': l_roi = val if key == 'pipe': pipeline = val if key == 'obj': stage = val str_cwd = pipeline.workingDir() log = stage.log() lst_statFunc = pipeline.l_statFunc() lst_group = pipeline.l_group() lst_pval = pipeline.l_pval() for roi in l_roi: log('Processing %s\n' % roi) for pipeline._str_statFunc in lst_statFunc: for pipeline._str_pval in lst_pval: for pipeline._str_surface in pipeline.l_surface(): for pipeline._str_hemi in pipeline.l_hemisphere(): str_summaryFileDir = '%s/%s/dty_analyze-3-cutoffs-%s-%s' % \ (roi, pipeline._str_statFunc, pipeline._str_pval, args.curvFunc) # print(str_summaryFileDir) os.chdir(str_summaryFileDir) for pipeline._str_group in lst_group: str_tagFile = 'separate-%s-AreaDensity-%s-%s.txt' % \ (pipeline._str_group, pipeline._str_hemi, pipeline._str_surface) if os.path.isfile(str_tagFile): for hit in [ line.strip() for line in open(str_tagFile, 'r') ]: hitFile = hit.split()[2] l_fileSpec = os.path.basename( hitFile).split('-') pipeline._str_hemi = l_fileSpec[2] pipeline._str_func = l_fileSpec[3] pipeline._str_surface = l_fileSpec[-3] if pipeline._str_func != 'thickness': ctype = 'curv' else: ctype = 'thickness' # print(os.getcwd()) # print(l_fileSpec) # log('hit = %s\n' % hit) curvMatch = [ s for s in pipeline.l_curvFunc() if pipeline._str_func == s ] if len(curvMatch): log('Logging hit for group %s, %s.%s.%s, p-val theshold %s, ctype %s\n' % \ ( pipeline._str_group, pipeline._str_hemi, pipeline._str_func, pipeline._str_surface, pipeline._str_pval, ctype)) try: misc.file_writeOnce('%s/%s/statFunc/%s' % \ (pipeline.dirSpec(), ctype, roi), '%s\n' % hit, mode='a') except: log('Ignoring %s-%s\n' % (pipeline._str_hemi, pipeline._str_surface)) os.chdir(str_cwd) return True