def f_stage2callback(**kwargs): str_cwd = os.getcwd() lst_subj = [] for key, val in kwargs.iteritems(): if key == 'subj': lst_subj = val if key == 'obj': stage = val for subj in lst_subj: # find the relevant input files in each <subj> dir os.chdir(subj) l_B0 = misc.find(_str_b0) l_ASL = misc.find(_str_asl) if not l_B0: error.fatal(pipe_hypothermia, 'noB0') if not l_ASL: error.fatal(pipe_hypothermia, 'noASL') _str_B0File = l_B0[0] _str_ASLFile = l_ASL[0] _str_outDir = 'outDir' _str_outFile = 'asl2b0.nii' misc.mkdir(_str_outDir) log = stage.log() log('Scheduling masconorm for "%s: ASL"..\n' % (subj)) str_cmd = 'masconorm.py --input %s/asl2b0.nii.gz --mask %s/b0Brain_mask.nii.gz --outStem %s/asl' % ( _str_outDir, _str_outDir, _str_outDir) #cluster = crun.crun_mosix() cluster = crun.crun() cluster.echo(False) cluster.echoStdOut(False) cluster.detach(False) cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True) if cluster.exitCode(): error.fatal(pipe_hypothermia, 'stageExec', cluster.stderr()) os.chdir(str_cwd) return True
def f_stage3callback(**kwargs): str_cwd = os.getcwd() lst_subj = [] for key, val in kwargs.iteritems(): if key == 'subj': lst_subj = val if key == 'obj': stage = val for subj in lst_subj: # find the relevant input files in each <subj> dir os.chdir(subj) l_ADC = misc.find(_str_adc) if not l_ADC: error.fatal(pipe_hypothermia, 'noADC') _str_ADCFile = l_ADC[0] _str_outDir = 'outDir' misc.mkdir(_str_outDir) log = stage.log() log('Scheduling masconorm for "%s: ADC"..\n' % (subj)) str_cmd = 'masconorm.py --input %s --mask %s/b0Brain_mask.nii.gz --outStem %s/adc' % ( _str_ADCFile, _str_outDir, _str_outDir) cluster = crun.crun() cluster.echo(False) cluster.echoStdOut(False) cluster.detach(False) cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True) os.chdir(str_cwd) return True
def f_stage0callback(**kwargs): str_cwd = os.getcwd() for key, val in kwargs.iteritems(): if key == 'obj': stage = val if key == 'pipe': pipeline = val log = stage.log() lst_annotation = pipeline.l_annotation() lst_pval = pipeline.l_pval() lst_group = pipeline.l_group() lst_hemi = pipeline.l_hemisphere() lst_surface = pipeline.l_surface() lst_statFunc = pipeline.l_statFunc() lst_ctype = ['thickness', 'curv'] lst_leaf = ['pval', 'statFunc', 'pval-only', 'statFunc-only', 'pval_N_statFunc'] for pipeline._str_annotation in lst_annotation: for pipeline._str_group in lst_group: for pipeline._str_pval in lst_pval: for pipeline._str_statFunc in lst_statFunc: for pipeline._str_surface in lst_surface: for pipeline._str_hemi in lst_hemi: for ctype in lst_ctype: for leaf in lst_leaf: OSshell('mkdir -p %s/%s/%s' % ( pipeline.dirSpec(), ctype, leaf)) stage.exitCode(0) return True
def f_stage0callback(**kwargs): str_cwd = os.getcwd() lst_subj = [] for key, val in kwargs.iteritems(): if key == 'subj': lst_subj = val if key == 'obj': stage = val for subj in lst_subj: # find the relevant input files in each <subj> dir os.chdir(subj) l_B0 = misc.find(_str_b0) if not l_B0: error.fatal(pipe_hypothermia, 'noB0') _str_B0File = l_B0[0] _str_outDir = '%s/outDir' % os.getcwd() _str_outFile = 'b0Brain.nii' misc.mkdir(_str_outDir) str_prefixCmd = '( cd %s ; ' % (os.getcwd()) log = stage.log() log('Scheduling brain extraction for "%s"...\n' % (subj)) str_cmd = 'bet.py --input %s --output %s/%s' % (_str_B0File, _str_outDir, _str_outFile) #cluster = crun.crun_mosix(cmdPrefix=str_prefixCmd) #cluster = crun.crun_mosix() cluster = crun.crun() #str_ccmd = cluster.echo(True) #log(str_ccmd) cluster.echo(False) cluster.echoStdOut(False) cluster.detach(False) cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True) if cluster.exitCode(): error.fatal(pipe_hypothermia, 'stageExec', cluster.stderr()) os.chdir(str_cwd) return True
def f_stage3callback(**kwargs): ''' Sorts intersect/xor of classes ''' str_cwd = os.getcwd() for key, val in kwargs.iteritems(): if key == 'roi': l_roi = val if key == 'obj': stage = val if key == 'pipe': pipeline = val str_cwd = pipeline.workingDir() log = stage.log() lst_pval = pipeline.l_pval() lst_group = pipeline.l_group() lst_hemi = pipeline.l_hemisphere() lst_surface = pipeline.l_surface() lst_statFunc = pipeline.l_statFunc() lst_roi = pipeline.l_roi() lst_ctype = ['thickness', 'curv'] lst_leaf = [ 'pval', 'statFunc', 'pval-only', 'statFunc-only', 'pval_N_statFunc' ] for pipeline._str_group in lst_group: for pipeline._str_pval in lst_pval: for pipeline._str_statFunc in lst_statFunc: for pipeline._str_surface in lst_surface: for pipeline._str_hemi in lst_hemi: for ctype in lst_ctype: log('Processing %s\n' % pipeline.dirSpecPartial()) os.chdir('%s/%s/%s' % (pipeline.dirSpec(), ctype, 'pval')) l_pval = OSshell('ls')[0].strip().split('\n') os.chdir( '%s/%s/%s' % (pipeline.dirSpec(), ctype, 'statFunc')) l_statFunc = OSshell('ls')[0].strip().split( '\n') os.chdir('../') if l_pval == ['']: sp = set() else: sp = set(l_pval) if l_statFunc == ['']: sf = set() else: sf = set(l_statFunc) U = sp.union(sf) I = sp.intersection(sf) pO = sp - I fO = sf - I for roi in list(I): log('Intersection ROI: %s\n' % roi) shutil.copyfile('statFunc/%s' % roi, 'pval_N_statFunc/%s' % roi) for roi in list(pO): log('p-val only ROI: %s\n' % roi) shutil.copyfile('pval/%s' % roi, 'pval-only/%s' % roi) for roi in list(fO): log('statFunc only ROI: %s\n' % roi) shutil.copyfile('statFunc/%s' % roi, 'statFunc-only/%s' % roi) stage.exitCode(0) return True
def f_stage2callback(**kwargs): ''' STAGE 2 ------- Run the (new) 2D cloud-based curvature analysis. ''' for key, val in kwargs.iteritems(): if key == 'obj': stage = val if key == 'pipe': pipeline = val os.chdir(pipeline.topDir()) log = stage.log() log('Stage 2 -- running from dir %s\n' % pipeline.topDir()) shellCluster = crun.crun_hpc_mosix(remoteUser='******', remoteHost='rc-majesty', schedulerStdOutDir='%s-%s' % (pipeline.schedulerStdOutDir(), '-stage-2'), schedulerStdErrDir='%s-%s' % (pipeline.schedulerStdErrDir(), '-stage-2')) for pipeline._str_annotation in pipeline.l_annotation(): for pipeline._str_subsample in pipeline.l_subsample(): ''' Read the ROI.lst file containing the various annotation ROIs, and for each element, start a separate deviation analysis process. ''' str_outDir = "%s%s/groupCurvAnalysis/%s" % \ ( pipeline.outDir(), pipeline._str_subsample, pipeline._str_annotation ) os.chdir("%s/%s" % (pipeline.topDir(), str_outDir)) with open('ROI.lst') as f: s = f.read() l_ROI = s.strip().split(' ') for str_ROI in l_ROI: for pipeline._str_hemi in pipeline.l_hemisphere(): for pipeline._str_surface in pipeline.l_surface(): str_execCmd = '%s/../../sh/%s -D %s/%s -h %s -s %s %s' % \ ( pipeline.topDir(), './ptile_deviation_analysis-raw.bash', pipeline.topDir(), str_outDir, pipeline._str_hemi, pipeline._str_surface, str_ROI ) print(os.getcwd()) log("Shell command = %s\n" % str_execCmd) shellCluster.waitForChild(True) shellCluster(str_execCmd) # Sleep for a second so as not to overload the head node # in tight scheduling loops time.sleep(1) shellCluster.blockOnChild() stage.exitCode(0) return True
def f_stage1callback(**kwargs): ''' Sorts intersect/xor of classes ''' str_cwd = os.getcwd() for key, val in kwargs.iteritems(): if key == 'obj': stage = val if key == 'pipe': pipeline = val log = stage.log() lst_annotation = pipeline.l_annotation() lst_subsample = pipeline.l_subsample() lst_pval = pipeline.l_pval() lst_group = pipeline.l_group() lst_hemi = pipeline.l_hemisphere() lst_surface = pipeline.l_surface() lst_statFunc = pipeline.l_statFunc() lst_ctype = pipeline.l_ctype() c = pipeline.dtree() for pipeline._str_annotation in lst_annotation: for pipeline._str_group in lst_group: for pipeline._str_pval in lst_pval: for pipeline._str_statFunc in lst_statFunc: for pipeline._str_surface in lst_surface: for pipeline._str_hemi in lst_hemi: for pipeline._str_ctype in lst_ctype: l_sum = [] d_occurence = {} for pipeline._str_subsample in lst_subsample: log('Processing %s %s for %s...' % (pipeline.dirSpecPartial(), pipeline._str_ctype, pipeline._str_subsample), lw=170) #print(pipeline.dirSpecSubsample()) os.chdir(pipeline.dirSpecSubsample()) l_roi = OSshell("cat all.tcl | grep label | awk '{print $2}'")[0].strip().split('\n') log('[ %02d ]\n' % len(l_roi), syslog=False, rw=20) #print(l_roi) c.cdnode(pipeline.dtreeDir()) c.touch('l_roi', l_roi) c.touch('path', pipeline.dtreeDir()) l_sum += l_roi c.cdnode('../') c.touch('l_sum', l_sum) l_uniq = sorted(set(l_sum)) for roi in l_uniq: d_occurence[roi] = float(l_sum.count(roi))/float(len(lst_subsample))*100 c.touch('d_occurence', d_occurence) os.chdir(pipeline.d2path()) # print(os.getcwd()) # print(json.dumps(d_occurence, sort_keys=True, indent=4, # separators=(',',': '))) json.dump(d_occurence, open('%s/occurence.txt' % os.getcwd(), "w"), sort_keys=True, indent=4, separators=(',',': ') ) stage.exitCode(0) return True
def f_stage4callback(**kwargs): ''' STAGE 4 ------- Run the ROI tagging. ''' for key, val in kwargs.iteritems(): if key == 'obj': stage = val if key == 'pipe': pipeline = val os.chdir(pipeline.topDir()) log = stage.log() log('Stage 4 -- running from dir %s\n' % pipeline.topDir()) shellCluster = crun.crun_hpc_mosix( remoteUser='******', remoteHost='rc-majesty', schedulerStdOutDir='%s-%s' % (pipeline.schedulerStdOutDir(), '-stage-4'), schedulerStdErrDir='%s-%s' % (pipeline.schedulerStdErrDir(), '-stage-4')) for pipeline._str_annotation in pipeline.l_annotation(): for pipeline._str_subsample in pipeline.l_subsample(): str_outDir = "%s%s/groupCurvAnalysis/%s" % \ ( pipeline.outDir(), pipeline._str_subsample, pipeline._str_annotation ) os.chdir("%s/%s" % (pipeline.topDir(), str_outDir)) str_pval = ','.join(pipeline.l_pval()) str_groups = ','.join(pipeline.l_group()) str_surface = ','.join(pipeline.l_surface()) str_statFunc = ','.join(pipeline.l_statFunc()) str_hemi = ','.join(pipeline.l_hemisphere()) str_curv = ','.join(pipeline.l_curvFunc()) str_execCmd = '%s --clobber --workingDir %s/%s --curvFunc %s --pval %s --group %s --surface %s --statFunc %s --hemi %s --stages 01234 --schedulerStdOutDir=%s --schedulerStdErrDir=%s $(cat ./ROI.lst)' % \ ( # pipeline.topDir(), str_outDir, "/neuro/users/rudolphpienaar/src/devel/roi_tag/roi_tag.py", pipeline.topDir(), str_outDir, str_curv, str_pval, str_groups, str_surface, str_statFunc, str_hemi, '/neuro/users/rudolphpienaar/scratch/%s-roi_tag.jobout' % os.getpid(), '/neuro/users/rudolphpienaar/scratch/%s-roi_tag.joberr' % os.getpid() ) log("Shell command = %s\n" % str_execCmd) shellCluster.waitForChild(True) shellCluster(str_execCmd) shellCluster.blockOnChild() stage.exitCode(0) return True
def f_stage3callback(**kwargs): ''' STAGE 3 ------- Run the overlap/density analysis in the cloud space. ''' for key, val in kwargs.iteritems(): if key == 'obj': stage = val if key == 'pipe': pipeline = val os.chdir(pipeline.topDir()) log = stage.log() log('Stage 3 -- running from dir %s\n' % pipeline.topDir()) shellCluster = crun.crun_hpc_mosix( remoteUser='******', remoteHost='rc-majesty', schedulerStdOutDir='%s-%s' % (pipeline.schedulerStdOutDir(), '-stage-3'), schedulerStdErrDir='%s-%s' % (pipeline.schedulerStdErrDir(), '-stage-3')) for pipeline._str_annotation in pipeline.l_annotation(): for pipeline._str_subsample in pipeline.l_subsample(): for pipeline._str_pval in pipeline.l_pval(): str_outDir = "%s%s/groupCurvAnalysis/%s" % \ ( pipeline.outDir(), pipeline._str_subsample, pipeline._str_annotation ) os.chdir("%s/%s" % (pipeline.topDir(), str_outDir)) str_groups = ','.join(pipeline.l_group()) str_curv = ','.join(pipeline.l_curvFunc()) for pipeline._str_hemi in pipeline.l_hemisphere(): for pipeline._str_surface in pipeline.l_surface(): str_execCmd = '%s/../../sh/%s -D %s/%s -p %s -g %s -h %s -c %s -s %s' % \ ( pipeline.topDir(), "./dty_analyze-here.bash", pipeline.topDir(), str_outDir, pipeline._str_pval, str_groups, pipeline._str_hemi, str_curv, pipeline._str_surface ) log("Current dir = %s\n" % os.getcwd()) log("Shell command = %s\n" % str_execCmd) shellCluster.waitForChild(True) shellCluster(str_execCmd) shellCluster.blockOnChild() stage.exitCode(0) return True
def f_stage1callback(**kwargs): ''' STAGE 1 ------- Run the resampler (intelligently) across all the sample experiment directories. ''' for key, val in kwargs.iteritems(): if key == 'obj': stage = val if key == 'pipe': pipeline = val os.chdir(pipeline.topDir()) log = stage.log() for pipeline._str_annotation in pipeline.l_annotation(): for pipeline._str_subsample in pipeline.l_subsample(): b_parentContainsSamples = False str_outDir = "%s%s" % \ ( pipeline.outDir(), pipeline._str_subsample, ) log('outDir = %s\n' % str_outDir) # Check if outParentDir contains samples l_numer = l_filterNumeric(os.listdir(str_outDir)) log('numerical directory count = %d\n' % len(l_numer)) if len(l_numer): b_parentContainsSamples = True str_stages = "12" else: str_stages = "012" str_subjList = ' '.join(pipeline._l_subj) str_execCmd = "%s -o %s -r %s -s %s -a %s %s" % \ ( "/neuro/users/rudolphpienaar/src/devel/roi_tag/rsample.py", str_outDir, pipeline._str_threshold, str_stages, pipeline._str_annotation, str_subjList ) log("Shell command = %s\n" % str_execCmd) OSshell(str_execCmd) stage.exitCode(0) return True
def f_stage0callback(**kwargs): lst_subj = [] for key, val in kwargs.iteritems(): if key == 'subj': lst_subj = val if key == 'obj': stage = val if key == 'pipe': pipeline = val lst_hemi = pipeline.l_hemisphere() lst_surface = pipeline.l_surface() lst_curv = pipeline.l_curv() if int(args.partitions) > _maxPartitions: error.fatal(hbwm, 'Partition') for pipeline._str_subj in lst_subj: for pipeline._str_hemi in lst_hemi: for pipeline._str_surface in lst_surface: for pipeline._str_curv in lst_curv: log = stage.log() log('Processing %s: %s.%s, %s...\n' % \ (pipeline.subj(), pipeline.hemi(), pipeline.surface(), pipeline.curv())) str_hostOnlySpec = '' if len(args.host): str_hostOnlySpec = "--host %s " % args.host log('Locking jobs to only run on host -->%s<--\n' % args.host) str_debug = "" if args.b_debug: str_debug = " --debug " str_queue = "" if args.queue: str_queue = " --queue %s " % args.queue str_cmd = "~/src/scripts/hbwm.py -v 10 -s %s %s -r -m %s -f %s -c %s -p %s --cluster %s %s %s %s" % \ (args.stages, str_hostOnlySpec, pipeline.hemi(), pipeline.surface(), pipeline.curv(), args.partitions, args.cluster, str_debug, str_queue, pipeline.subj()) print str_cmd shell = crun.crun() shell.echo(False) shell.echoStdOut(False) shell.detach(False) shell(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True) if shell.exitCode(): error.fatal(hbwm, 'stageExec', shell.stderr()) os.chdir(pipeline.startDir()) return True
def f_fileCheck(**kwargs): ''' A file exist checker. ''' stage = None l_file = [] for key, val in kwargs.iteritems(): if key == 'obj': stage = val if key == 'files': l_file = val log = stage.log() b_allFilesExist = True for str_fileName in l_file: log('Checking on %s...' % str_fileName, lw=85) b_exists = misc.file_exists(str_fileName) b_allFilesExist &= b_exists log('[ %s ]\n' % b_exists, rw=20, syslog=False) return b_allFilesExist
def f_stage1callback(**kwargs): for key, val in kwargs.iteritems(): if key == 'obj': stage = val if key == 'pipe': pipeline = val log = stage.log() lst_subj = pipeline.l_subj() os.chdir(pipeline.outDir()) misc.mkdir('groupCurvAnalysis') os.chdir('groupCurvAnalysis') misc.mkdir(pipeline.annotationDir()) os.chdir(pipeline.dirSpecAnnot(base='orig')) for bashFile in glob.glob(r'*.bash'): shutil.copy(bashFile, pipeline.dirSpecAnnot(base='rsampled')) for lstFile in glob.glob(r'*.lst'): shutil.copy(lstFile, pipeline.dirSpecAnnot(base='rsampled')) str_CMDoriginalROI = "find . -maxdepth 1 -mindepth 1 -type d | grep -v ROItag | awk -F\/ '{print $2}'" pipeline.l_ROI(OSshell(str_CMDoriginalROI)[0].strip().split('\n')) lst_ROI = pipeline.l_ROI() os.chdir(pipeline.dirSpecAnnot(base='rsampled')) # print(os.getcwd()) total = 0 for pipeline._str_ROI in lst_ROI: os.chdir(pipeline.dirSpecAnnot(base='rsampled')) log('Creating ROI dir %s\n' % (pipeline._str_ROI)) misc.mkdir(pipeline._str_ROI) os.chdir(pipeline._str_ROI) tagCentroidCMD = 'ls %s/%s/*centroid*txt' % (pipeline.dirSpecAnnot( base='orig'), pipeline._str_ROI) pipeline.l_centroidTXT( OSshell(tagCentroidCMD)[0].strip().split('\n')) for pipeline._str_centroidTXT in pipeline.l_centroidTXT(): log('\tCopying original sample %s\n' % os.path.basename(pipeline._str_centroidTXT)) shutil.copy( pipeline._str_centroidTXT, os.path.join(pipeline.outDir(), 'groupCurvAnalysis', pipeline.annotationDir(), pipeline._str_ROI)) total += 1 stage.exitCode(0) return True
def f_stage2callback(**kwargs): for key, val in kwargs.iteritems(): if key == 'obj': stage = val if key == 'pipe': pipeline = val log = stage.log() lst_subj = pipeline.l_subj() os.chdir(pipeline.outDir()) str_CMDsubj = "find . -maxdepth 1 -mindepth 1 -type l | awk -F\/ '{print $2}' | sort " pipeline.l_subjResampled(OSshell(str_CMDsubj)[0].strip().split('\n')) lst_subjResampled = pipeline.l_subjResampled() lst_subjResampled.insert(0, 'Subj') os.chdir(pipeline.dirSpecAnnot(base='rsampled')) str_CMDoriginalROI = "find . -maxdepth 1 -mindepth 1 -type d | grep -v ROItag | awk -F\/ '{print $2}'" pipeline.l_ROI(OSshell(str_CMDoriginalROI)[0].strip().split('\n')) lst_ROI = pipeline.l_ROI() total = 0 for pipeline._str_ROI in lst_ROI: os.chdir( '%s/%s' % (pipeline.dirSpecAnnot(base='rsampled'), pipeline._str_ROI)) str_CMDcentroid = "ls -1 *centroid*txt" pipeline.l_centroidTXT( OSshell(str_CMDcentroid)[0].strip().split('\n')) log('Switching to ROI %s\n' % pipeline._str_ROI) for pipeline._str_centroidTXT in pipeline.l_centroidTXT(): log('\tProcessing %s\n' % pipeline._str_centroidTXT) l_meas = [ line.strip() for line in open(pipeline._str_centroidTXT, 'r') ] filt = [x for x in l_meas if x.split()[0] in lst_subjResampled] FILEfilt = open('%s.filt' % pipeline._str_centroidTXT, 'w') for line in filt: FILEfilt.write(' %s\n' % line) # log('Saved in %s\n' % os.getcwd()) FILEfilt.close() shutil.move('%s.filt' % pipeline._str_centroidTXT, pipeline._str_centroidTXT) total += 1 stage.exitCode(0) return True
def f_stage0callback(**kwargs): str_cwd = os.getcwd() for key, val in kwargs.iteritems(): if key == 'roi': l_roi = val if key == 'obj': stage = val if key == 'pipe': pipeline = val str_cwd = pipeline.workingDir() log = stage.log() lst_pval = pipeline.l_pval() lst_group = pipeline.l_group() lst_hemi = pipeline.l_hemisphere() lst_surface = pipeline.l_surface() lst_statFunc = pipeline.l_statFunc() lst_roi = pipeline.l_roi() lst_ctype = pipeline.l_curvFunc() lst_ctype = ['thickness', 'curv'] lst_leaf = [ 'pval', 'statFunc', 'pval-only', 'statFunc-only', 'pval_N_statFunc' ] os.chdir(str_cwd) if os.path.isdir(pipeline.outDir()) and not pipeline.clobber(): log('Existing outDir tree found... deleting...\n') shutil.rmtree(pipeline.outDir()) OSshell('mkdir -p %s' % pipeline.outDir()) os.chdir(pipeline.outDir()) pipeline.outDir(os.getcwd()) os.chdir(str_cwd) if OSshell.exitCode() != 0: error.fatal(pipeline, 'outDirNotCreate') for pipeline._str_group in lst_group: for pipeline._str_pval in lst_pval: for pipeline._str_statFunc in lst_statFunc: for pipeline._str_surface in lst_surface: for pipeline._str_hemi in lst_hemi: for ctype in lst_ctype: for leaf in lst_leaf: OSshell('mkdir -p %s/%s/%s' % (pipeline.dirSpec(), ctype, leaf)) stage.exitCode(0) return True
def f_stage0callback(**kwargs): for key, val in kwargs.iteritems(): if key == 'obj': stage = val if key == 'pipe': pipeline = val log = stage.log() lst_subj = pipeline.l_subj() if os.path.isdir(pipeline.outDir()): log('Existing outDir tree found... deleting...\n') shutil.rmtree(pipeline.outDir()) log('Creating output directory...\n') OSshell('mkdir -p %s' % pipeline.outDir()) print(OSshell.stdout()) if OSshell.exitCode() != 0: error.fatal(pipeline, 'outDirNotCreate') os.chdir(pipeline.outDir()) pipeline.outDir(os.getcwd()) passcount = 0 failcount = 0 total = 0 for pipeline._str_subj in lst_subj: f_cutoff = random.random() log('thresholding %s...' % pipeline._str_subj) if f_cutoff < pipeline.threshold(): OSshell('ln -s %s/%s .' % (pipeline.topDir(), pipeline._str_subj)) log('[ passed ]\n', rw=20, syslog=False) passcount += 1 else: log('[ failed ]\n', rw=20, syslog=False) failcount += 1 total += 1 f_passPerc = float(passcount) / float(total) * 100 f_failPerc = float(failcount) / float(total) * 100 log('passed: %d (%5.2f%s)\n' % (passcount, f_passPerc, '%')) log('failed: %d (%5.2f%s)\n' % (failcount, f_failPerc, '%')) stage.exitCode(0) return True
def f_stage0callback(**kwargs): for key, val in kwargs.iteritems(): if key == 'jobs': jobs = val if key == 'obj': stage = val if key == 'pipe': pipeline = val # Create shell for scheduling/executing on the remote HPC pipeline.stageShell_createRemoteInstance(args.cluster, stage=stage) for job in range(0, int(args.jobs)): log = stage.log() log('Processing job: %d...\n' % job) str_cmd = args.cmd shell = stage.shell() shell.echo(True) shell( str_cmd, waitForChild=shell.waitForChild(), stdoutflush=True, stderrflush=True ) if shell.exitCode(): error.fatal(hpc, 'stageExec', shell.stderr()) os.chdir(pipeline.startDir()) return True
def f_stage4callback(**kwargs): str_cwd = os.getcwd() lst_subj = [] for key, val in kwargs.iteritems(): if key == 'subj': lst_subj = val if key == 'obj': stage = val for subj in lst_subj: # find the relevant input files in each <subj> dir _str_outDir = 'outDir' misc.mkdir(_str_outDir) os.chdir("%s/outDir" % subj) log = stage.log() log('Scheduling MatLAB basac analysis for subject "%s"...\n' % (subj)) str_cmd = 'eval "/chb/pices/arch/x86_64-Linux/bin/matlab -nodesktop -nosplash -nojvm -r \\\"c = basac_drive(\'%s\'); exit\\\"" |tee basac_process.log' % ( os.getcwd() ) cluster = crun.crun() cluster.echo(False) cluster.echoStdOut(False) cluster.detach(False) cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True) os.chdir(str_cwd) return True
def f_stage0callback(**kwargs): ''' STAGE 0 ------- Build output directory trees for the analysis. ''' str_cwd = os.getcwd() for key, val in kwargs.iteritems(): if key == 'obj': stage = val if key == 'pipe': pipeline = val log = stage.log() b_alreadyResampled = False b_outAnnotDirExist = False for pipeline._str_annotation in pipeline.l_annotation(): for pipeline._str_subsample in pipeline.l_subsample(): b_parentContainsSamples = False str_outDir = "%s%s/groupCurvAnalysis/%s" % \ ( pipeline.outDir(), pipeline._str_subsample, pipeline._str_annotation ) str_outParentDir = "/".join(str_outDir.split('/')[0:-2]) log('Building output tree for "%s", annotation "%s"\n' % \ ( str_outParentDir, pipeline._str_annotation)) b_outAnnotDirExist = os.path.isdir(str_outDir) if not b_outAnnotDirExist: misc.mkdir(str_outDir) # Check if outParentDir contains samples l_numer = l_filterNumeric(os.listdir(str_outParentDir)) if len(l_numer): b_parentContainsSamples = True stage.exitCode(0) return True
def f_stage4callback(**kwargs): str_cwd = os.getcwd() for key, val in kwargs.iteritems(): if key == 'roi': l_roi = val if key == 'obj': stage = val if key == 'pipe': pipeline = val str_cwd = pipeline.workingDir() log = stage.log() lst_pval = pipeline.l_pval() lst_group = pipeline.l_group() lst_hemi = pipeline.l_hemisphere() lst_surface = pipeline.l_surface() lst_statFunc = pipeline.l_statFunc() lst_roi = pipeline.l_roi() lst_ctype = ['thickness', 'curv'] lst_leaf = ['pval', 'statFunc', 'pval-statFunc'] shellCluster = crun.crun_hpc_mosix( remoteUser='******', remoteHost='rc-majesty', schedulerStdOutDir='%s-%s' % (pipeline.schedulerStdOutDir(), '-stage-4'), schedulerStdErrDir='%s-%s' % (pipeline.schedulerStdErrDir(), '-stage-4')) str_curvFunc = ','.join(pipeline.l_curvFunc()) log('Group list: %s\n' % lst_group) for pipeline._str_group in lst_group: for pipeline._str_pval in lst_pval: for pipeline._str_statFunc in lst_statFunc: for pipeline._str_surface in lst_surface: for pipeline._str_hemi in lst_hemi: for ctype in lst_ctype: if ctype == 'thickness': str_curvFunc = 'thickness' else: l_curv = list(pipeline.l_curvFunc()) log('curvFunc = %s\n' % l_curv) if any('thickness' in s for s in l_curv): l_curv.remove('thickness') str_curvFunc = ','.join(l_curv) log('Processing %s-%s-%s\n' % \ (pipeline._str_group, pipeline.dirSpecPartial(), ctype)) os.chdir( '%s/%s/%s' % (pipeline.dirSpec(), ctype, 'pval-only')) sortCmd = "/bin/ls -l | sort -n -k 5 | awk '{print $9}'" l_pval = OSshell(sortCmd)[0].strip().split( '\n') l_pval = filter(None, l_pval) os.chdir('%s/%s/%s' % (pipeline.dirSpec(), ctype, 'statFunc-only')) l_statFunc = OSshell(sortCmd)[0].strip().split( '\n') l_statFunc = filter(None, l_statFunc) os.chdir('%s/%s/%s' % (pipeline.dirSpec(), ctype, 'pval_N_statFunc')) l_pNs = OSshell(sortCmd)[0].strip().split('\n') l_pNs = filter(None, l_pNs) os.chdir('../') labelCount = 0 rLabelCount = 0 colorMult = 9 misc.file_writeOnce( 'pval-only.freeview.txt', '#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF' % pipeline._str_hemi) misc.file_writeOnce( 'statFunc-only.freeview.txt', '#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF' % pipeline._str_hemi) misc.file_writeOnce( 'pval_N_statFunc.freeview.txt', '#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF' % pipeline._str_hemi) misc.file_writeOnce( 'all.freeview.txt', '#!/bin/bash\n\nSUBJNAME=$1; SURF=$2\nexport SUBJECT=$SUBJECTS_DIR/$SUBJNAME\nfreeview --surface $SUBJECT/surf/%s.$SURF' % pipeline._str_hemi) for pvalROI in l_pval: if pvalROI != 'entire': intensity = 127 + labelCount * colorMult if intensity > 255: intensity = 255 misc.file_writeOnce( 'pval-only.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=%d,0,0' % (pipeline._str_hemi, pvalROI, intensity), mode='a') misc.file_writeOnce( 'pval-only.tcl', 'labl_load %s.%s.label ; labl_set_color %d %d 0 0\n' % (pipeline._str_hemi, pvalROI, labelCount, intensity), mode='a') misc.file_writeOnce( 'all.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=%d,0,0' % (pipeline._str_hemi, pvalROI, intensity), mode='a') misc.file_writeOnce( 'all.tcl', 'labl_load %s.%s.label ; labl_set_color %d %d 0 0\n' % (pipeline._str_hemi, pvalROI, labelCount, intensity), mode='a') labelCount += 1 rLabelCount = labelCount misc.file_writeOnce( 'pval-only.tcl', tcl_append(pipeline.namespec(), '%s-pval-only' % str_curvFunc), mode='a') for statFuncROI in l_statFunc: if statFuncROI != 'entire': intensity = 127 + ( labelCount - rLabelCount) * colorMult if intensity > 255: intensity = 255 misc.file_writeOnce( 'statFunc-only.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=0,0,%d' % (pipeline._str_hemi, statFuncROI, intensity), mode='a') misc.file_writeOnce( 'statFunc-only.tcl', 'labl_load %s.%s.label ; labl_set_color %d 0 0 %d\n' % (pipeline._str_hemi, statFuncROI, labelCount, intensity), mode='a') misc.file_writeOnce( 'all.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=0,0,%d' % (pipeline._str_hemi, statFuncROI, intensity), mode='a') misc.file_writeOnce( 'all.tcl', 'labl_load %s.%s.label ; labl_set_color %d 0 0 %d\n' % (pipeline._str_hemi, statFuncROI, labelCount, intensity), mode='a') labelCount += 1 rLabelCount = labelCount misc.file_writeOnce( 'statFunc-only.tcl', tcl_append( pipeline.namespec(), '%s-statFunc-only' % str_curvFunc), mode='a') for pNsROI in l_pNs: if pNsROI != 'entire': intensity = 127 + ( labelCount - rLabelCount) * colorMult if intensity > 255: intensity = 255 misc.file_writeOnce( 'pval_N_statFunc.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=0,%d,0' % (pipeline._str_hemi, pNsROI, intensity), mode='a') misc.file_writeOnce( 'pval_N_statFunc.tcl', 'labl_load %s.%s.label ; labl_set_color %d 0 %d 0\n' % (pipeline._str_hemi, pNsROI, labelCount, intensity), mode='a') misc.file_writeOnce( 'all.freeview.txt', ':label=$SUBJECT/label/%s.%s.label:label_color=0,%d,0' % (pipeline._str_hemi, pNsROI, intensity), mode='a') misc.file_writeOnce( 'all.tcl', 'labl_load %s.%s.label ; labl_set_color %d 0 %d 0\n' % (pipeline._str_hemi, pNsROI, labelCount, intensity), mode='a') labelCount += 1 misc.file_writeOnce( 'pval_N_statFunc.tcl', tcl_append( pipeline.namespec(), '%s-pval_N_statFunc' % str_curvFunc), mode='a') misc.file_writeOnce( 'all.tcl', tcl_append(pipeline.namespec(), '%s-all' % str_curvFunc), mode='a') # Now run the actual tcl files to generate the tiffs. log("Current dir = %s\n" % os.getcwd()) str_scriptDir = '/neuro/users/rudolphpienaar/projects/dyslexia-curv-analysis-2/sh' str_execCmd = 'cd %s; %s/%s -D %s -h %s' % \ ( os.getcwd(), str_scriptDir, # pipeline.workingDir(), "./tksurfer-run.bash", os.getcwd(), pipeline._str_hemi, ) log("Shell command = %s\n" % str_execCmd) shellCluster.waitForChild(True) shellCluster.echoStdOut(True) shellCluster.echoStdErr(True) shellCluster(str_execCmd) shellCluster.blockOnChild() stage.exitCode(0) return True
def f_stage2callback(**kwargs): ''' Builds list of statFunc tagged hits ''' str_cwd = os.getcwd() l_roi = [] for key, val in kwargs.iteritems(): if key == 'roi': l_roi = val if key == 'pipe': pipeline = val if key == 'obj': stage = val str_cwd = pipeline.workingDir() log = stage.log() lst_statFunc = pipeline.l_statFunc() lst_group = pipeline.l_group() lst_pval = pipeline.l_pval() for roi in l_roi: log('Processing %s\n' % roi) for pipeline._str_statFunc in lst_statFunc: for pipeline._str_pval in lst_pval: for pipeline._str_surface in pipeline.l_surface(): for pipeline._str_hemi in pipeline.l_hemisphere(): str_summaryFileDir = '%s/%s/dty_analyze-3-cutoffs-%s-%s' % \ (roi, pipeline._str_statFunc, pipeline._str_pval, args.curvFunc) # print(str_summaryFileDir) os.chdir(str_summaryFileDir) for pipeline._str_group in lst_group: str_tagFile = 'separate-%s-AreaDensity-%s-%s.txt' % \ (pipeline._str_group, pipeline._str_hemi, pipeline._str_surface) if os.path.isfile(str_tagFile): for hit in [ line.strip() for line in open(str_tagFile, 'r') ]: hitFile = hit.split()[2] l_fileSpec = os.path.basename( hitFile).split('-') pipeline._str_hemi = l_fileSpec[2] pipeline._str_func = l_fileSpec[3] pipeline._str_surface = l_fileSpec[-3] if pipeline._str_func != 'thickness': ctype = 'curv' else: ctype = 'thickness' # print(os.getcwd()) # print(l_fileSpec) # log('hit = %s\n' % hit) curvMatch = [ s for s in pipeline.l_curvFunc() if pipeline._str_func == s ] if len(curvMatch): log('Logging hit for group %s, %s.%s.%s, p-val theshold %s, ctype %s\n' % \ ( pipeline._str_group, pipeline._str_hemi, pipeline._str_func, pipeline._str_surface, pipeline._str_pval, ctype)) try: misc.file_writeOnce('%s/%s/statFunc/%s' % \ (pipeline.dirSpec(), ctype, roi), '%s\n' % hit, mode='a') except: log('Ignoring %s-%s\n' % (pipeline._str_hemi, pipeline._str_surface)) os.chdir(str_cwd) return True