def printTimeStats(self, name, timelist): if len(timelist) < 2: return meantime = self.stats['timesum']/float(self.stats['count']) timearray = numpy.array(timelist, dtype=numpy.float64) apDisplay.printColor("%s: %s (%.2f percent)"% (name, apDisplay.timeString(timearray.mean(), timearray.std()), 100*timearray.mean()/meantime), "blue")
def runKerdenSOM(self, indata): """ From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/KerDenSOM KerDenSOM stands for "Kernel Probability Density Estimator Self-Organizing Map". It maps a set of high dimensional input vectors into a two-dimensional grid. """ apDisplay.printMsg("Running KerDen SOM") outstamp = os.path.join(self.params['rundir'], self.timestamp) kerdencmd = ( "xmipp_classify_kerdensom -verb 1 -i %s -o %s -xdim %d -ydim %d -saveclusters "% (indata, outstamp, self.params['xdim'], self.params['ydim']) ) ### convergence criteria if self.params['converge'] == "fast": kerdencmd += " -eps 1e-5 " elif self.params['converge'] == "slow": kerdencmd += " -eps 1e-9 " else: kerdencmd += " -eps 1e-7 " apDisplay.printColor(kerdencmd, "cyan") proc = subprocess.Popen(kerdencmd, shell=True) proc.wait() time.sleep(1) return
def boxParticlesFromImage(self, imgdata, partdatas, shiftdata): ### convert database particle data to coordinates and write boxfile boxfile = os.path.join(self.params['rundir'], imgdata['filename']+".box") parttree, boxedpartdatas = apBoxer.processParticleData(imgdata, self.boxsize, partdatas, shiftdata, boxfile, rotate=self.params['rotate']) if self.params['boxfiles']: ### quit and return, boxfile created, now process next image return None, None, None ### check if we have particles again if len(partdatas) == 0 or len(parttree) == 0: apDisplay.printColor(self.shortname+" has no remaining particles and has been rejected\n","cyan") return None, None, None ### set up output file path imgstackfile = os.path.join(self.params['rundir'], self.shortname+".hed") if (self.is_dd_stack and (self.params['nframe'] or self.params['driftlimit']) and not self.params['phaseflipped'] and not self.params['rotate']): # If processing on whole image is not needed, it is more efficient to use mmap to box frame stack apDisplay.printMsg("boxing "+str(len(parttree))+" particles into temp file: "+imgstackfile) framelist = self.dd.getFrameList(self.params) apBoxer.boxerFrameStack(self.dd.framestackpath, parttree, imgstackfile, self.boxsize, framelist) else: self._boxParticlesFromImage(imgdata, parttree, imgstackfile) partmeantree = self.calculateParticleStackStats(imgstackfile, boxedpartdatas) imgstackfile = self.postProcessParticleStack(imgdata, imgstackfile, boxedpartdatas, len(parttree)) return boxedpartdatas, imgstackfile, partmeantree
def createNewSession(self): apDisplay.printColor("Creating a new session", "cyan") ### get user data userdata = self.getUserData() sessionq = leginon.leginondata.SessionData() sessionq['name'] = self.params['sessionname'] sessionq['image path'] = self.leginonimagedir sessionq['frame path'] = self.leginonframedir sessionq['comment'] = self.params['description'] sessionq['user'] = userdata sessionq['hidden'] = False projectdata = leginon.projectdata.projects.direct_query(self.params['projectid']) projectexpq = leginon.projectdata.projectexperiments() projectexpq['project'] = projectdata projectexpq['session'] = sessionq if self.params['commit'] is True: projectexpq.insert() self.sessiondata = sessionq apDisplay.printColor("Created new session %s"%(self.params['sessionname']), "cyan") return
def medianVolume(self): volpath = os.path.join(self.params['rundir'], "volumes/*a.mrc") mrcfiles = glob.glob(volpath) volumes = [] for filename in mrcfiles: if os.path.isfile(filename): vol = mrc.read(filename) print filename, vol.shape volumes.append(vol) volarray = numpy.asarray(volumes, dtype=numpy.float32) try: medarray = numpy.median(volarray, axis=0) except: medarray = numpy.median(volarray) medfile = os.path.join(self.params['rundir'], "volumes/medianVolume.mrc") print medfile, medarray.shape mrc.write(medarray, medfile) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) uploadcmd = ( ("uploadModel.py --projectid=%d --session=%s --file=%s " +"--apix=%.3f --sym=%s --name=satmedian-recon%d.mrc --res=30 --description='%s %d'") %(self.params['projectid'], sessiondata['name'], medfile, apix, self.params['symmname'], self.params['reconid'], "SAT selected median volume for recon", self.params['reconid'], ) ) apDisplay.printColor(uploadcmd, "purple") f = open("upload.sh", "w") f.write(uploadcmd+"\n") f.close()
def createSpiderFile(self): """ takes the stack file and creates a spider file ready for processing """ emancmd = "proc2d " if not os.path.isfile(self.stack['file']): apDisplay.printError("stackfile does not exist: "+self.stack['file']) emancmd += self.stack['file']+" " spiderstack = os.path.join(self.params['rundir'], "start.spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack+" " emancmd += "apix="+str(self.stack['apix'])+" " if self.params['lowpass'] > 0: emancmd += "lp="+str(self.params['lowpass'])+" " if self.params['highpass'] > 0: emancmd += "hp="+str(self.params['highpass'])+" " if self.params['bin'] > 1: clipboxsize = self.boxsize*self.params['bin'] emancmd += "shrink="+str(self.params['bin'])+" " emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" " emancmd += "last="+str(self.params['numpart']-1)+" " emancmd += "spider edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan") return spiderstack
def postLoopFunctions(self): ### Delete CTF corrected images if self.params['keepall'] is False: pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*.dwn.mrc') apFile.removeFilePattern(pattern) ### remove Ace2 images pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*mrc.corrected.mrc') apFile.removeFilePattern(pattern) ### remove Spider images if self.params['fliptype'] == 'spiderimage': pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*_out.spi') apFile.removeFilePattern(pattern) pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*_tf.spi') apFile.removeFilePattern(pattern) if self.noimages is True: return stackpath = os.path.join(self.params['rundir'], self.params['single']) ### delete this after testing apStack.averageStack(stack = stackpath) ### Create Stack Mean Plot if self.params['commit'] is True and self.params['meanplot'] is True: stackid = apStack.getStackIdFromPath(stackpath) if stackid is not None: apStackMeanPlot.makeStackMeanPlot(stackid) apDisplay.printColor("Timing stats", "blue") self.printTimeStats("Batch Boxer", self.batchboxertimes) self.printTimeStats("Ctf Correction", self.ctftimes) self.printTimeStats("Stack Merging", self.mergestacktimes) self.printTimeStats("Mean/Std Read", self.meanreadtimes) self.printTimeStats("DB Insertion", self.insertdbtimes)
def getDiffResForOverfocus(radii=None, cs=2e-3, volts=120000): """ given Cs and kV, determine the initial resolution where the difference between overfocus and underfocus is clearly visible. value returned in Angstroms, but radii must be in meters """ if debug is True: print "getDiffResForOverfocus()" if debug is True: apDisplay.printColor("getDiffRes radii: 1/%.2fA --> 1/%.2fA"%(1/radii[1]*1e10, 1/radii[-1]*1e10), "cyan") t0 = time.time() checkParams(focus1=1.0e-6, focus2=1.0e-6, cs=cs, volts=volts, ampconst=0.0, failParams=False) lamb = ctftools.getTEMLambda(volts) s = radii pi = math.pi csgamma = 2*pi*0.25*cs*(lamb**3)*(s**4) #over/under-focus difference is visible when Cs component is greater than 0.05 index = numpy.searchsorted(csgamma, 0.03) diffres = 1.0/radii[index-1]*1e10 apDisplay.printColor("Overfocus/Underfocus difference resolution is: 1/%.2fA"%(diffres), "cyan") if debug is True: print "difference resolution complete in %.9f sec"%(time.time()-t0) return diffres
def insertAlignParticlesIntoDatabase(self, level): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") D=self.getClassificationAtLevel(level) for ref in D: ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = ref+1 refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] if ref in self.resdict: refq['ssnr_resolution'] = self.resdict[ref] ### setup particle info ... NOTE: ALIGNMENT PARAMETERS ARE NOT SAVED IN XMIPP 2.4 for partnum in D[ref]: alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = int(partnum)+1 alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle(self.runparams['stackid'], int(partnum)+1) ### particle numbering starts with 0!!!!!!! alignpartq['stackpart'] = stackpartdata alignpartq['ref'] = refq ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in " +apDisplay.timeString(time.time()-t0), "cyan") return
def commonLines(stackfile, maskdiam=None, minfreq=0, maxfreq=0.25, ang=5.0, numiter=200, outdocfile=None, numpart=None, dataext=".spi"): """ performs common lines on a input spider stack """ if numpart is None or numpart < 3: apDisplay.printError("undefined number of particles") if maskdiam is None: apDisplay.printError("undefined mask diameter") starttime = time.time() if dataext in stackfile: stackfile = stackfile[:-4] randdocfile = generateRandomAngles(numpart) mySpider = spyder.SpiderSession(dataext=dataext, logo=True) mySpider.toSpider("OP", stackfile+"@*****", #stack file "1-%d"%(numpart), #number of particles str(maskdiam), #mark diameter "%.3f,%.3f"%(minfreq,maxfreq), #frequency range for line projections str(ang), #angular increment or accuracy str(numiter), #number of iterations randdocfile, #random angles doc file outdocfile, #output angles doc file ) mySpider.close() apDisplay.printColor("finished common lines in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def initializeRefinementUploadVariables(self): """ untars results, reads required parameters, establishes necessary objects for upload script """ apDisplay.printColor("uploading refinement results for %s reconstruction routine" % (self.package), "cyan") ### establish directories and make an appion results directory self.basepath = os.path.abspath(self.params["rundir"]) self.resultspath = os.path.abspath(os.path.join(self.params["rundir"], str(self.package) + "_results")) if not os.path.isdir(self.resultspath): os.mkdir(self.resultspath) self.reconpath = os.path.abspath( os.path.join(self.params["rundir"], self.runparams["reconstruction_working_dir"]) ) ### get all stack parameters, map particles in reconstruction to particles in stack, get all model data self.stackdata = apStack.getOnlyStackData(self.runparams["stackid"]) self.stackmapping = apRecon.partnum2defid(self.runparams["stackid"]) self.modeldata = [] if len(self.runparams["modelid"].split(",")) > 1: models = self.runparams["modelid"].split(",") for i in range(len(models)): self.modeldata.append(appiondata.ApInitialModelData.direct_query(int(models[i]))) else: self.modeldata.append(appiondata.ApInitialModelData.direct_query(self.runparams["modelid"])) return
def rctParticleShift(volfile, origstackfile, eulerdocfile, iternum, numpart, pixrad, timestamp, dataext=".spi"): """ inputs: stack, in spider format eulerdocfile outputs: volume """ starttime = time.time() ### create corresponding projections projstackfile = "projstack%s-%03d.spi"%(timestamp, iternum) projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext) ### clean up files ccdocfile = "ccdocfile%s-%03d.spi"%(timestamp, iternum) apFile.removeFile(ccdocfile) alignstackfile = "alignstack%s-%03d.spi"%(timestamp, iternum) apFile.removeFile(alignstackfile) ### align particles to projection apDisplay.printMsg("Shifting particles") crossCorrelateAndShift(origstackfile, projstackfile, alignstackfile, ccdocfile, numpart) if not os.path.isfile(alignstackfile): apDisplay.printError("aligned stack file not found: "+alignstackfile) apDisplay.printColor("finished correlations in "+apDisplay.timeString(time.time()-starttime), "cyan") return alignstackfile
def getResolutionData(self, avgpath, iternum): evenvol = os.path.join(avgpath, "avglist3_%dp1.mrc" % (self.params["rescut"])) oddvol = os.path.join(avgpath, "avglist3_%dp2.mrc" % (self.params["rescut"])) evenhed = os.path.join(avgpath, "avglist3_%dp1.hed" % (self.params["rescut"])) oddhed = os.path.join(avgpath, "avglist3_%dp2.hed" % (self.params["rescut"])) emancmd1 = "proc3d %s %s" % (evenvol, evenhed) apEMAN.executeEmanCmd(emancmd1, verbose=True) emancmd2 = "proc3d %s %s" % (oddvol, oddhed) apEMAN.executeEmanCmd(emancmd2, verbose=True) fscfile = "fsc.eotest.%d" % (iternum) fscpath = os.path.join(avgpath, fscfile) emancmd3 = "proc3d %s %s fsc=%s" % (evenhed, oddhed, fscpath) apEMAN.executeEmanCmd(emancmd3, verbose=True) if not os.path.isfile(fscpath): apDisplay.printWarning("Could not find FSC file: " + fscpath) return None f = open(fscpath, "r") xy = f.readlines() lines = len(xy) boxsize = lines * 2.0 f.close() # calculate the resolution: halfres = apRecon.calcRes(fscpath, boxsize, self.params["step"]) apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms" % (halfres), "cyan") # save to database resq = appiondata.ApResolutionData() resq["half"] = halfres resq["fscfile"] = fscpath return resq
def backprojectCG(stackfile, eulerdocfile, volfile, numpart, pixrad, dataext=".spi"): """ inputs: stack, in spider format eulerdocfile outputs: volume """ ### setup starttime = time.time() stackfile = spyder.fileFilter(stackfile) eulerdocfile = spyder.fileFilter(eulerdocfile) volfile = spyder.fileFilter(volfile) if not os.path.isfile(stackfile+dataext): apDisplay.printError("stack file not found: "+stackfile+dataext) if not os.path.isfile(eulerdocfile+dataext): apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext) apFile.removeFile(volfile+dataext) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("BP CG", stackfile+"@*****", #stack file "1-%d"%(numpart), #number of particles str(pixrad), #particle radius eulerdocfile, #angle doc file "N", #has symmetry?, does not work volfile, #filename for volume "%.1e,%.1f" % (1.0e-5, 0.0), #error, chi^2 limits "%d,%d" % (25,1), #iterations, 1st derivative mode "2000", #lambda - higher=less sensitive to noise ) mySpider.close() apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def start(self): print 'wait=',self.params['wait'] max_loop_num_trials = 60 * 3 wait_time = 20 self.last_num_stacks = 0 if self.params['wait']: num_trials = 0 while True: limit_reached = self.loopCheckAndProcess() if limit_reached: apDisplay.printMsg('image limit reached. Stoping...') break if self.num_stacks <= self.last_num_stacks: if num_trials >= max_loop_num_trials: apDisplay.printColor('Checked for stack file %d times. Finishing....' % max_loop_num_trials,'magenta') apDisplay.printMsg('Rerun this script if you know more are coming') break else: num_trials += 1 else: # reset trial number if new stack is found num_trials = 0 apDisplay.printColor('Finished stack file checking in rundir. Will check again in %d seconds' % wait_time,'magenta') time.sleep(wait_time) self.last_num_stacks = self.num_stacks else: self.loopCheckAndProcess()
def backproject3F(stackfile, eulerdocfile, volfile, numpart, dataext=".spi"): """ inputs: stack, in spider format eulerdocfile outputs: volume """ ### setup starttime = time.time() stackfile = spyder.fileFilter(stackfile) eulerdocfile = spyder.fileFilter(eulerdocfile) volfile = spyder.fileFilter(volfile) if not os.path.isfile(stackfile+dataext): apDisplay.printError("stack file not found: "+stackfile+dataext) if not os.path.isfile(eulerdocfile+dataext): apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext) apFile.removeFile(volfile+dataext) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("BP 3F", stackfile+"@*****", #stack file "1-%d"%(numpart), #number of particles eulerdocfile, #angle doc file "*", #input symmetry file, '*' for skip volfile, #filename for volume ) mySpider.close() apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def start(self): """ this is the main component of the script where all the processing is done """ ### initialize some variables self.runq = None self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) apDisplay.printMsg("Pixel size: %.5f"%(self.apix)) self.boxsize = apStack.getStackBoxsize(self.params['stackid']) apDisplay.printMsg("Box size: %d"%(self.boxsize)) self.checkResults() self.stackmapping = apRecon.partnum2defid(self.params['stackid']) self.numiter = self.getNumberOfIterations() for i in range(self.numiter): iternum = i+1 apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green") self.uploadIteration(iternum) reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir']) if reconrunid: apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid']) apRecon.setGoodBadParticlesFromReconId(reconrunid) else: apDisplay.printWarning("Could not find recon run id")
def getNumberOfIterations(self): iternum = 0 stop = False while stop is False: ## check if iteration is complete iternum += 1 paramfile = "params.iter%03d.par"%(iternum) if not os.path.isfile(paramfile): apDisplay.printWarning("Parameter file %s is missing"%(paramfile)) stop = True break imagicvolume = "threed.%03da.hed"%(iternum) if not os.path.isfile(imagicvolume): apDisplay.printWarning("Volume file %s is missing"%(imagicvolume)) stop = True break combineshell = "iter%03d/frealign.iter%03d.combine.sh"%(iternum, iternum) if not os.path.isfile(combineshell): apDisplay.printWarning("Shell file %s is missing"%(combineshell)) stop = True break ### set last working iteration numiter = iternum-1 if numiter < 1: apDisplay.printError("No iterations were found") apDisplay.printColor("Found %d complete iterations"%(numiter), "green") return numiter
def findLastCompletedIteration(self): #recondir = os.path.join(self.params['rundir'], "recon") recondir = self.params['rundir'] iternum = 0 stop = False while stop is False: ## check if iteration is complete iternum += 1 class1Volume = "recon_it%03d_half1_model.star"%(iternum) class2Volume = "recon_it%03d_half2_model.star"%(iternum) class1Volumepath = os.path.join(recondir,class1Volume) class2Volumepath = os.path.join(recondir,class2Volume) if not os.path.isfile(class1Volumepath) or not os.path.isfile(class2Volumepath): apDisplay.printWarning("Model.star file %s or %s is missing"%(class1Volumepath, class2Volumepath)) stop = True break ### set last working iteration numiter = iternum-1 if numiter < 1: apDisplay.printError("No iterations were found") apDisplay.printColor("Found %d complete iterations"%(numiter), "green") return numiter
def start(self): ### universal particle counter self.partnum = 1 ### final stack file self.combinefile = os.path.join( self.params['rundir'], self.params['stackfilename'] ) if os.path.isfile(self.combinefile): apDisplay.printError("A stack with name "+self.params['stackfilename']+" and path " +self.params['rundir']+" already exists.") ### loop through stacks for stackstr in self.params['stackids']: stackid = int(stackstr) ### get stack data stackdata = apStack.getOnlyStackData(stackid) ### append particle to stack file self.appendToStack(stackdata) if self.params['commit'] is True: ### insert stack data apDisplay.printColor("inserting new stack particles from stackid="+str(stackid), "cyan") self.commitStack(stackid) else: apDisplay.printWarning("not committing data to database") apStack.averageStack(stack=self.combinefile)
def convertStackToSpider(self, emanstackfile, classnum): """ takes the stack file and creates a spider file ready for processing """ if not os.path.isfile(emanstackfile): apDisplay.printError("stackfile does not exist: "+emanstackfile) ### first high pass filter particles apDisplay.printMsg("pre-filtering particles") apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid']) emancmd = ("proc2d "+emanstackfile+" "+emanstackfile +" apix="+str(apix)+" hp="+str(self.params['highpasspart']) +" inplace") apEMAN.executeEmanCmd(emancmd, verbose=True) ### convert imagic stack to spider emancmd = "proc2d " emancmd += emanstackfile+" " spiderstack = os.path.join(self.params['rundir'], str(classnum), "otrstack"+self.timestamp+".spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack+" " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan") return spiderstack
def runEoTest(self, corrSelectOdd, corrSelectEven, cnum, apshstack, apsheuler, iternum): apshOddVolfile = os.path.join(self.params['rundir'], str(cnum), "apshVolume_Odd-%03d.spi"%(iternum)) apshEvenVolfile = os.path.join(self.params['rundir'], str(cnum), "apshVolume_Even-%03d.spi"%(iternum)) self.APSHbackProject(apshstack, apsheuler, apshOddVolfile, cnum, corrSelectOdd) self.APSHbackProject(apshstack, apsheuler, apshEvenVolfile, cnum, corrSelectEven) fscout = os.path.join(self.params['rundir'], str(cnum), "FSCout-%03d.spi"%(iternum)) backproject.calcFSC(apshOddVolfile, apshEvenVolfile, fscout) ### Calculate FSC - taken from Neil's RCT script apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] emancmd = "proc3d %s %s"%(apshEvenVolfile, apshEvenVolfile+".mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) emancmd = "proc3d %s %s"%(apshOddVolfile, apshOddVolfile+".mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) fscfile = os.path.join(self.params['rundir'], "fscdata%s.fsc"%(self.timestamp)) emancmd = "proc3d %s %s fsc=%s"%(apshEvenVolfile+".mrc", apshOddVolfile+".mrc", fscfile) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) if not os.path.isfile(fscfile): apDisplay.printError("Even-Odd fsc calculation failed") boxsize = self.getBoxSize() self.fscresolution = apRecon.getResolutionFromFSCFile(fscfile, boxsize, apix, msg=True) apDisplay.printColor( ("Final FSC resolution: %.5f" % (self.fscresolution)), "cyan") return fscout
def checkConflicts(self): ### setup correct database after we have read the project id if 'projectid' in self.params and self.params['projectid'] is not None: apDisplay.printMsg("Using split database") # use a project database newdbname = apProject.getAppionDBFromProjectId(self.params['projectid']) sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '"+newdbname+"'", "green") # DD processes self.dd = apDDprocess.DDStackProcessing() print self.dd # get stack data self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid']) self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True) self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) # query image qimage = self.stackparts[0]['particle']['image'] # DD info self.dd.setImageData(qimage) self.dd.setDDStackRun(self.params['ddstackid']) self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def refineEllipseLoop(self, fftarray, lowerbound, upperbound): """ refines the following parameters, in order ellipRatio ellipAngle defocus amplitude contrast (mostly a dependent variable) """ self.fminCount = 0 ### "self.ellipseParams" controls what self.from2Dinto1D() does ellipRatio = self.ellipseParams['a']/self.ellipseParams['b'] ellipAlpha = self.ellipseParams['alpha'] origEllipParams = copy.deepcopy(self.ellipseParams) self.fftarray = fftarray self.lowerbound = lowerbound self.upperbound = upperbound ### create function self.refineMinFunc that would return res80+res50 x0 = [ellipRatio, ellipAlpha] maxfun = self.params['refineIter'] results = scipy.optimize.fmin(self.refineMinFunc, x0=x0, maxfun=maxfun) ellipRatio, ellipAlpha = results apDisplay.printColor("BEST FROM FMIN :: defRatio=%.3f < a=%.2f" %(ellipRatio**2, math.degrees(ellipAlpha)), "blue") self.printBestValues() time.sleep(10)
def setDBfromProjectId(projectid, die=True): newdbname = getAppionDBFromProjectId(projectid, die=die) if newdbname is None: return False sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '"+newdbname+"'", "green") return True
def projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext=".spi"): """ project 3D volumes using given Euler angles """ starttime = time.time() volfile = spyder.fileFilter(volfile) eulerdocfile = spyder.fileFilter(eulerdocfile) projstackfile = spyder.fileFilter(projstackfile) if not os.path.isfile(volfile+dataext): apDisplay.printError("volume file not found: "+volfile+dataext) if not os.path.isfile(eulerdocfile+dataext): apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext) apFile.removeFile(projstackfile) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("PJ 3Q", volfile, #input vol file str(pixrad), #pixel radius "1-%d"%(numpart), #number of particles eulerdocfile, #Euler DOC file projstackfile+"@*****", #output projections ) mySpider.close() apDisplay.printColor("finished projections in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def _startLoop(self, info): """ appionLoop OVERRIDE initilizes several parameters for a new image and checks if it is okay to start processing image """ if info is None: self.stats['lastimageskipped'] = True self.stats['skipcount'] += 1 return False name = info['filename'] # check to see if image of the same name is already in leginon imgq = leginon.leginondata.AcquisitionImageData(session=self.session, filename=name) results = imgq.query(readimages=False) if results: apDisplay.printWarning("File %s.mrc exists at the destination" % name) apDisplay.printWarning("Skip Uploading") self.stats['lastimageskipped'] = True self.stats['skipcount'] += 1 return False #calc images left self.stats['imagesleft'] = self.stats['imagecount'] - self.stats['count'] #only if an image was processed last if(self.stats['lastcount'] != self.stats['count']): apDisplay.printColor( "\nStarting image "+str(self.stats['count'])\ +" ( skip:"+str(self.stats['skipcount'])+", remain:"\ +str(self.stats['imagesleft'])+" ) file: "\ +apDisplay.short(name), "green") self.stats['lastcount'] = self.stats['count'] self._checkMemLeak() # check to see if image has already been processed if self._alreadyProcessed(info): return False self.stats['waittime'] = 0 return True
def runChimeraScript(chimscript, xvfb=False): if not chimscript or not os.path.isfile(chimscript): print chimscript apDisplay.printError("Could not find file: apChimSnapshot.py") #apDisplay.printColor("Trying to use chimera for model imaging","cyan") if xvfb is True: port = apParam.resetVirtualFrameBuffer() time.sleep(1) if 'CHIMERA' in os.environ and os.path.isdir(os.environ['CHIMERA']): chimpath = os.environ['CHIMERA'] os.environ['CHIMERA'] = chimpath os.environ['CHIMERAPATH'] = os.path.join(chimpath,"share") os.environ['LD_LIBRARY_PATH'] = os.path.join(chimpath,"lib")+":"+os.environ['LD_LIBRARY_PATH'] chimexe = os.path.join(chimpath,"bin/chimera") if not os.path.isfile(chimexe): apDisplay.printWarning("Could not find chimera at: "+chimexe) else: chimpath = None chimexe = "chimera" #apDisplay.printWarning("'CHIMERA' environmental variable is unset") rendercmd = (chimexe+" --debug python:"+chimscript) logf = open("chimeraRun.log", "a") apDisplay.printColor("running Chimera:\n "+rendercmd, "cyan") if xvfb is True: print "import -verbose -display :%d -window root screencapture.png"%(port) proc = subprocess.Popen(rendercmd, shell=True, stdout=logf, stderr=logf) proc.wait() logf.close() if xvfb is True: apParam.killVirtualFrameBuffer(port) return
def normalizeVol(volfile, dataext=".spi"): """ inputs: volume outputs: volume """ ### setup starttime = time.time() volfile = spyder.fileFilter(volfile) if not os.path.isfile(volfile+dataext): apDisplay.printError("volume file not found: "+volfile+dataext) mySpider = spyder.SpiderSession(dataext=dataext, logo=True) ### read out the statistics of the volume mySpider.toSpider("FS x11,x12", volfile, #filename for volume ) mySpider.toSpider("IF(x12.LT.0.0)x12=-x12") ### set all values to positive mySpider.toSpider("AR", volfile, #filename for volume "_1", "(P1+x12)", ) ### save file mySpider.toSpider("CP", "_1", volfile, #filename for volume ) mySpider.close() apDisplay.printColor("finished normalizing the volume to set all values to be positive"+apDisplay.timeString(time.time()-starttime), "cyan") return
def createSpiderFile(self): """ takes the stack file and creates a spider file ready for processing """ emancmd = "proc2d " if not os.path.isfile(self.stack["file"]): apDisplay.printError("stackfile does not exist: " + self.stack["file"]) emancmd += self.stack["file"] + " " spiderstack = os.path.join(self.params["rundir"], "start.spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack + " " emancmd += "apix=" + str(self.stack["apix"]) + " " if self.params["lowpass"] > 0: emancmd += "lp=" + str(self.params["lowpass"]) + " " emancmd += "last=" + str(self.params["numpart"] - 1) + " " emancmd += "shrink=" + str(self.params["bin"]) + " " clipsize = int(math.floor(self.stack["boxsize"] / self.params["bin"] / 2.0) * self.params["bin"] * 2) emancmd += "clip=" + str(clipsize) + "," + str(clipsize) + " " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished eman in " + apDisplay.timeString(time.time() - starttime), "cyan") return spiderstack
def createIterationJobRefineFiles(self, oldprefix, newprefix, stackFile): """ Create multiple job files for frealign reconstruction using the mpiexec command """ ### create individual mpi scripts particlePerProcess = float( self.numpart) / self.params['totalprocs'] - 1 apDisplay.printColor( "Using approx %.1f particles per process file, for a total of %d processes" % (particlePerProcess, self.params['totalprocs']), "purple") lastPart = 0 procjobfiles = [] for i in range(self.params['totalprocs']): procNum = i + 1 inputVolFile = "../threed.%s.mrc" % (oldprefix) firstPart = lastPart + 1 lastPart = firstPart + particlePerProcess if lastPart > self.numpart - 2: lastPart = self.numpart inputParamFile = "../params.%s.par" % (oldprefix) apParam.createDirectory(newprefix, warning=False) jobFile = "%s/refine.%s.proc%03d.sh" % (newprefix, newprefix, procNum) procPrefix = "%s.proc%03d" % (newprefix, procNum) f = open(jobFile, 'w') f.write("#!/bin/sh\n\n") fullpath = os.path.join(self.params['rundir'], newprefix) f.write('cd %s\n' % fullpath) f.write("/bin/rm -fv frealign.%s.out\n" % (procPrefix)) f.write("/bin/rm -fv outparams.%s.par\n" % (procPrefix)) f.write("/bin/rm -fv shift.%s.par\n" % (procPrefix)) f.write("mkdir /tmp/frealign\n") f.write("rsync -vrtPhL %s /tmp/frealign/%s\n" % (inputVolFile, os.path.basename(inputVolFile))) f.write("rsync -vrtPhL %s /tmp/frealign/%s\n" % (stackFile, os.path.basename(stackFile))) f.close() #partDiff = math.floor(lastPart) - math.floor(firstPart) #print "proc %d: %.1f->%.1f (%d)"%(procNum, firstPart, lastPart, partDiff) self.appendFrealignJobFile(jobFile, inputParamFile, inputVolFile, stackFile, math.floor(firstPart), math.floor(lastPart), procPrefix, recon=False) return
def applyEnvelopeAndCTF(self, stack): ### get defocus lists numpart = self.params['projcount'] cut = int(numpart/80.0)+1 apDisplay.printMsg("%d particles per dot"%(cut)) if len(self.deflist1) == 0: self.getListOfDefoci(numpart) ### break up particles partlistdocfile = apXmipp.breakupStackIntoSingleFiles(stack, filetype="mrc") t0 = time.time() apDisplay.printMsg("Applying CTF and Envelop to particles") ### apply CTF using ACE2 ctfapplydocfile = self.applyCTFToDocFile(partlistdocfile) ### apply Envelop using ACE2 envelopdocfile = self.applyEnvelopToDocFile(ctfapplydocfile) ### correct CTF using ACE2 if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True: ctfcorrectdocfile = self.correctCTFToDocFile(envelopdocfile) else: ctfcorrectdocfile = envelopdocfile timeper = (time.time()-t0)/float(numpart) apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green") apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green") ### write corrected particle list to doc file ctfpartlist = [] ctfpartlistfile = os.path.join(self.params['rundir'], "ctfpartlist.lst") inf = open(ctfcorrectdocfile, 'r') outf = open(ctfpartlistfile, "w") for line in inf: ### get filename filename = line.strip().split()[0] if not os.path.isfile(filename): apDisplay.printError("CTF and envelop apply failed") ctfpartlist.append(filename) outf.write(filename+"\t1\n") inf.close() outf.close() ### merge individual files into a common stack ctfstack = os.path.join(self.params['rundir'], "ctfstack.hed") apXmipp.gatherSingleFilesIntoStack(ctfpartlistfile, ctfstack, filetype="mrc") if self.params['pad'] is True: emancmd = "proc2d %s %s.clip.hed clip=%d,%d" % (ctfstack, ctfstack[:-4], self.params['box'], self.params['box']) apParam.runCmd(emancmd, "EMAN") shutil.move("%s.clip.hed" % ctfstack[:-4], "%s.hed" % ctfstack[:-4]) shutil.move("%s.clip.img" % ctfstack[:-4], "%s.img" % ctfstack[:-4]) return ctfstack, ctfpartlist
def evenLogSplit(self, start, end, power=1.7): endlog = int(round(math.log(end) / math.log(power), 0)) startlog = int(round(math.log(start) / math.log(power), 0)) stacklist = [] for n in range(startlog, endlog, 1): numparticles = round(math.pow(power, n), 0) stacklist.append(int(numparticles)) apDisplay.printColor( "Making stacks of the following sizes: " + str(stacklist), "cyan") return (stacklist)
def start(self): res = apRecon.getResolutionFromFSCFile(self.params['fscfile'], self.params['boxsize'], self.params['apix'], self.params['criteria']) apDisplay.printColor(("resolution: %.5f at criteria %.3f" % (res, self.params['criteria'])), "cyan") sys.stdout.write( "%s resolution %.5f (%.3f)\n" % (self.params['fscfile'], res, self.params['criteria']))
def runIMAGICclassify(self): bfile = "msaclassify.job" outfile = "classes" apFile.removeStack(outfile) numIters = int(self.params['numpart']*self.params['itermult']) decrement = self.params['start']-self.params['end'] if self.params['iter']>0: decrement /= float(self.params['iter']) numClasses = self.params['start']-(decrement*self.params['currentiter']) stackfile = self.params['alignedstack'] self.params['currentnumclasses'] = numClasses f = open(bfile,'w') f.write("#!/bin/csh -f\n") f.write("setenv IMAGIC_BATCH 1\n") f.write("%s/msa/classify.e <<EOF\n" % self.imagicroot) f.write("IMAGES\n") f.write("%s\n"%stackfile) f.write("0\n") f.write("%i\n"%self.params['activeeigen']) f.write("YES\n") f.write("%i\n"%numClasses) f.write("classes_start\n") f.write("EOF\n") f.write("%s/msa/classum.e <<EOF\n" % self.imagicroot) f.write("%s\n"%stackfile) f.write("classes_start\n") f.write("%s\n"%outfile) f.write("YES\n") f.write("NONE\n") f.write("0\n") if int(self.imagicversion) >= 120619: f.write("NONE\n") # Mode of summing statistics f.write("EOF\n") ## make eigenimage stack appion-compatible f.write("proc2d eigenim.hed eigenim.hed inplace\n") f.write("touch msaclassify_done.txt\n") f.write("exit\n") f.close() ## execute the batch file aligntime0 = time.time() apEMAN.executeEmanCmd("chmod 755 "+bfile) apDisplay.printColor("Running IMAGIC .batch file: %s"%(os.path.abspath(bfile)), "cyan") apIMAGIC.executeImagicBatchFile(os.path.abspath(bfile)) os.remove("msaclassify_done.txt") if not os.path.exists(outfile+".hed"): apDisplay.printError("ERROR IN IMAGIC SUBROUTINE") apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-aligntime0), "cyan")
def insertParticlesIntoDatabase(self, stackid, partList): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in partList: count += 1 if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = partdict['refnum'] if refq['refnum'] < 1: apDisplay.printError("Classes must start with 1") refq['iteration'] = self.lastiter #refq['mrcfile'] = refbase+".mrc" ### need to create mrcs of each class??? refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] if partdict['refnum'] in self.resdict: refq['ssnr_resolution'] = self.resdict[partdict['refnum']] refq['isac_generation'] = partdict['generation'] ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partdict['partnum'] if alignpartq['partnum'] < 1: apDisplay.printError("Particles must start with 1") alignpartq['alignstack'] = self.alignstackdata if self.params['alignstackid'] is None: stackpartdata = apStack.getStackParticle( stackid, partdict['origPartNum']) else: stackpartdata = apStack.getStackParticleFromAlignParticle( self.params['alignstackid'], partdict['origPartNum']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['inplane'] alignpartq['mirror'] = partdict['mirror'] alignpartq['ref'] = refq alignpartq['score'] = partdict['peak'] ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(count) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan") return
def processImage(self, imgdata): #image = self.getImage(imgdata, self.params['bin']) self.image = imgdata['image'] # Check the image shape imgshape = numpy.asarray(imgdata['image'].shape) apDisplay.printMsg("MRC Image Shape prior to processing:") print imgshape imagepath = os.path.join(imgdata['session']['image path'], imgdata['filename'] + ".mrc") # Convert the MRC image to a jpg for find_mask.py apDisplay.printMsg("Converting mrc image to jpg.") jpg_dir = os.path.join(self.params['rundir'], "jpgs") jpg_image = os.path.join(jpg_dir, imgdata['filename'] + ".jpg") apParam.createDirectory(jpg_dir, warning=False) pyami.numpil.write(self.image, jpg_image) if (self.params['test']): self.outfile = os.path.join(self.params['rundir'], "tests", imgdata['filename'] + "_mask.jpg") else: self.outfile = os.path.join(self.params['rundir'], "masks", imgdata['filename'] + "_mask.jpg") downsample = str(self.params['downsample']) compsizethresh = str(self.params['compsizethresh']) adapthresh = str(self.params['adapthresh']) dilation = str(self.params['dilation']) erosion = str(self.params['erosion']) blur = str(self.params['blur']) options = " --downsample=" + downsample + " --compsizethresh=" + compsizethresh + " --adapthresh=" + adapthresh + " --blur=" + blur + " --dilation=" + dilation + " --erosion=" + erosion commandline = ("source " + self.activatepath + "; python `which find_mask.py` --ifile=" + jpg_image + " --ofile=" + self.outfile + options + "\n") # Test with test image #commandline = ( "source /opt/em_hole_finder/env/bin/activate; python /opt/em_hole_finder/find_mask_amber.py \n" ) ### run command apDisplay.printMsg("running em hole finder " + time.asctime()) apDisplay.printColor(commandline, "purple") if True: proc = subprocess.Popen(commandline, shell=True) else: outf = open("automasker.out", "a") errf = open("automasker.err", "a") proc = subprocess.Popen(commandline, shell=True, stderr=errf, stdout=outf) proc.wait()
def rejectImage(self, imgdata): shortname = apDisplay.short(imgdata['filename']) if self.params['mag']: if not apDatabase.checkMag(imgdata, self.params['mag']): apDisplay.printColor( shortname + " was not at the specific magnification", "cyan") return False return True
def onClose(self): if self.params['fulltomoId']: apDisplay.printMsg('------------------------') apDisplay.printWarning( 'Repeat the same script to create more subtomogram') apDisplay.printColor( 'etomo_subrecon.py --session=%s --projectid=%d --fulltomoid=%d --description="" --commit --expId=%d --jobtype=%s --runname=etomosub' % (self.params['sessionname'], self.params['projectid'], self.params['fulltomoId'], self.params['expid'], 'etomo_subrecon'), 'cyan') apDisplay.printMsg('------------------------')
def estimateIterTime(self): secperiter = 0.12037 calctime = ((self.params['numpart'] / 1000.0) * (self.stack['boxsize'] / self.params['bin'])**2 / self.params['angle']**2 / float(self.nproc) * secperiter) if self.params['mirror'] is True: calctime *= 2.0 self.params['estimatedtime'] = calctime apDisplay.printColor( "Estimated first iteration time: " + apDisplay.timeString(calctime), "purple")
def insertParticlesIntoDatabase(self, partlist, insert=False): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for particle in partlist: count += 1 ### get all particle parameters partnum = count rotation = particle[0] * -1 ### in spider negative rotation is clockwise shiftx = particle[1] shifty = particle[2] ccc = particle[3] refnum = int(particle[4]) mirror = particle[5] if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = refnum refq['iteration'] = self.params['numiter'] refq['imagicfile'] = "references.hed" refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['templatestack'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId']) reffile = os.path.join(self.params['rundir'], refq['imagicfile']) if not os.path.isfile(reffile): apDisplay.printError("could not find reference file: "+reffile) ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partnum alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle(self.params['stackId'], partnum) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = shiftx alignpartq['yshift'] = shifty alignpartq['rotation'] = rotation alignpartq['mirror'] = mirror alignpartq['ref'] = refq alignpartq['correlation'] = ccc ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in " +apDisplay.timeString(time.time()-t0), "cyan") return
def generateCTF1d(radii=None, focus=1.0e-6, cs=2e-3, volts=120000, ampconst=0.07, failParams=False, overfocus=False): """ calculates a CTF function based on the input details Use SI units: meters, radians, volts Underfocus is postive (defocused) """ if debug is True: print "generateCTF1dFromRadii()" if radii is None: radii = generateRadii1d(numpoints=256, pixelsize=1e-10) if debug is True: apDisplay.printColor( "generateCTF radii: 1/%.2fA --> 1/%.2fA" % (1 / radii[1] * 1e10, 1 / radii[-1] * 1e10), "cyan") t0 = time.time() checkParams(focus1=focus, focus2=focus, cs=cs, volts=volts, ampconst=ampconst, failParams=failParams) lamb = ctftools.getTEMLambda(volts) s = radii pi = math.pi if overfocus is True: focus = -1.0 * focus gamma = -0.5 * pi * cs * (lamb**3) * (s**4) + pi * focus * lamb * (s**2) if overfocus is True: gamma = -1.0 * gamma A = ampconst B = math.sqrt(1.0 - ampconst**2) prectf = A * numpy.cos(gamma) + B * numpy.sin(gamma) ctf = prectf**2 if debug is True: print "generate 1D ctf complete in %.9f sec" % (time.time() - t0) return ctf
def setFileName(self): if self.params['name'] is None: ### assign provided name basename = "emdb%s-%s" % (self.params['emdbid'], self.timestamp) else: ### clean up provided name basename = os.path.splitext(os.path.basename( self.params['name']))[0] self.params['name'] = os.path.join(self.params['rundir'], basename) apDisplay.printColor("Naming EMDB model: " + self.params['name'], "cyan") return
def close(self): self.onClose() loadavg = os.getloadavg()[0] if loadavg > 2.0: apDisplay.printMsg("Load average is high "+str(round(loadavg,2))) time.sleep(loadavg**2) apParam.closeFunctionLog(functionname=self.functionname, logfile=self.logfile, msg=(not self.quiet)) if self.quiet is False: apDisplay.printMsg("Ended at "+time.strftime("%a, %d %b %Y %H:%M:%S")) apDisplay.printMsg("Memory increase during run: %.3f MB"%((mem.active()-self.startmem)/1024.0)) apDisplay.printColor("Total run time:\t"+apDisplay.timeString(time.time()-self.t0),"green")
def satAverageCmd(self): keepfile = os.path.join(self.params['rundir'], "keeplist-tot" + self.datastr + ".lst") newname = "recon%d_cut%d_iter%d.hed" % ( self.params['reconid'], self.params['cutrange'] * 10, self.iternum) cmd = ("satAverage.py " + " --projectid=" + str(self.params['projectid']) + " --reconid=" + str(self.params['reconid']) + " \\\n --mask=62 --iter=" + str(self.iternum) + " \\\n --stackname=" + newname + " \\\n --keep-list=" + keepfile + " \n") print "New satAverage.py Command:" apDisplay.printColor(cmd, "purple")
def oldLogSplit(self, start,end,divisions): end=math.log(end) start=math.log(start) incr=(end-start)/divisions val=start stacklist=[] for n in range(0, divisions): nptcls=int(round(math.exp(val))) stacklist.append(nptcls) val+=incr apDisplay.printColor("Making stacks of the following sizes: "+str(stacklist), "cyan") return(stacklist)
def onClose(self): if self.fullrundata: apDisplay.printMsg('------------------------') apDisplay.printWarning( 'To create full tomogram reconstruction and commit the result to database with these sampled tomograms, you need to use etomo_recon.py to start eTOMO and continue at "Tomogram Positioning" in %s with the .edf file by running this AppionScript:' ) apDisplay.printColor( 'etomo_recon.py --session=%s --projectid=%d --samplerunid=%d --description="" --commit --expId=%d --jobtype=%s' % (self.params['sessionname'], self.params['projectid'], self.fullrundata.dbid, self.params['expid'], 'etomo_recon'), 'cyan') apDisplay.printMsg('------------------------')
def readFileToPeakTree(self): apDisplay.printMsg("Reading file: " + self.params['filename']) f = open(self.params['filename'], "r") count = 0 imgfilename2peaklist = {} apDisplay.printMsg("Reading input file") for line in f: ### format: x <tab> y <tab> filename sline = line.strip() cols = sline.split('\t') ### must have 3 columns if len(cols) == 4: partid, xcoord, ycoord, filename = cols elif len(cols) == 3: xcoord, ycoord, filename = cols else: continue ### check to make sure our x,y are integers, if not skip to next line in file try: xcoord = int(xcoord) ycoord = int(ycoord) except: continue ### create new list for new files if not filename in imgfilename2peaklist.keys(): imgfilename2peaklist[filename] = [] peakdict = { 'diameter': self.params['diam'], 'xcoord': xcoord, 'ycoord': ycoord, 'peakarea': 10, } count += 1 if count % 10 == 0: sys.stderr.write(".") imgfilename2peaklist[filename].append(peakdict) sys.stderr.write("done\n") f.close() apDisplay.printColor( "Found %d particles in %d images" % (count, len(imgfilename2peaklist.keys())), "cyan") if count == 0: apDisplay.printError("No particles were found") return imgfilename2peaklist
def onClose(self): if self.fulltomodata: apDisplay.printMsg('------------------------') apDisplay.printWarning( 'To create sub tomogram reconstruction and commit the result to database with this full tomogram, you need to use etomo_subrecon.py to start eTOMO and continue at "Post-Processing" with the .edf file by running this AppionScript:' ) apDisplay.printColor( 'etomo_subrecon.py --session=%s --projectid=%d --fulltomoid=%d --description="" --commit --expId=%d --jobtype=%s --runname=etomosub' % (self.params['sessionname'], self.params['projectid'], self.fulltomodata.dbid, self.params['expid'], 'etomo_subrecon'), 'cyan') apDisplay.printMsg('------------------------')
def runrefine(self): ### setup Xmipp command recontime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params['rundir'], self.partlistdocfile) + " -vol " + os.path.join(self.params['rundir'], self.voldocfile) + " -iter " + str(self.params['maxiter']) + " -o " + os.path.join(self.params['rundir'], "part" + self.timestamp) + " -psi_step " + str(self.params['psi']) + " -ang " + str(self.params['phi'])) ### fast mode if self.params['fast'] is True: xmippopts += " -fast " if self.params['fastmode'] == "narrow": xmippopts += " -C 1e-10 " elif self.params['fastmode'] == "wide": xmippopts += " -C 1e-18 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " ### normalization if self.params['norm'] is True: xmippopts += " -norm " ### symmetry if self.params['symmetry'] is not None: xmippopts += " -sym " + self.params['symmetry'] + " " if self.nproc > 1 and self.mpirun is not None: ### use multi-processor apDisplay.printColor("Using " + str(self.nproc) + " processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_ml_refine3d", die=True) mpiruncmd = self.mpirun + " -np " + str( self.nproc) + " " + xmippexe + " " + xmippopts self.writeXmippLog(mpiruncmd) apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True) else: ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_refine3d", die=True) xmippcmd = xmippexe + " " + xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) apDisplay.printMsg("Reconstruction time: " + apDisplay.timeString(time.time() - recontime))
def start(self): self.iflag = 1 self.setIBLOW() ### get stack info self.stackdata = apStack.getOnlyStackData(self.params['stackid']) self.refinestackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) apImagicFile.setImagic4DHeader(self.refinestackfile) ### copy stack or start job file if self.params['cluster'] is False: #create alias to stack data pass if self.params['cluster'] is True: self.setupMultiNode() if self.params['reconstackid'] is not None: self.reconstackdata = apStack.getOnlyStackData( self.params['stackid']) self.reconstackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) else: self.reconstackfile = self.refinestackfile ### create initial model file self.currentvol = os.path.basename(self.setupInitialModel()) ### create parameter file self.currentparam = os.path.basename(self.setupParticleParams()) apDisplay.printColor( "Initial files:\n Stack: %s\n Volume: %s\n Params: %s\n" % (os.path.basename( self.refinestackfile), self.currentvol, self.currentparam), "violet") ## run frealign for number for refinement cycles for i in range(self.params['numiter']): iternum = i + 1 if self.params['cluster'] is True: self.multiNodeRun(iternum) else: self.singleNodeRun(iternum) time.sleep(2) ### calculate FSC #emancmd = 'proc3d %s %s fsc=fsc.eotest.%d' % (evenvol, oddvol, self.params['iter']) #apEMAN.executeEmanCmd(emancmd, verbose=True) if self.params['cluster'] is True: self.prepareForCluster() print "Done!"
def convertDefociToConvention(ctfvalues): if debug is True: apDisplay.printColor( "Final params: def1: %.2e | def2: %.2e | angle: %.1f" % (ctfvalues['defocus1'], ctfvalues['defocus2'], ctfvalues['angle_astigmatism']), "cyan") # amplitude contrast must be btw 0.0 and 0.5 # sometimes we get a slightly negative number from ACE1, see bug #2003 if abs(ctfvalues['amplitude_contrast']) < 0.005: ctfvalues['amplitude_contrast'] = 0.0 # program specific corrections? angle = ctfvalues['angle_astigmatism'] #by convention: abs(ctfvalues['defocus1']) < abs(ctfvalues['defocus2']) if abs(ctfvalues['defocus1']) > abs(ctfvalues['defocus2']): # incorrect, need to shift angle by 90 degrees apDisplay.printWarning("|def1| > |def2|, flipping defocus axes") defocus1 = ctfvalues['defocus2'] defocus2 = ctfvalues['defocus1'] angle += 90 else: # correct, ratio > 1 defocus1 = ctfvalues['defocus1'] defocus2 = ctfvalues['defocus2'] if defocus1 < 0 and defocus2 < 0: apDisplay.printWarning( "Negative defocus values, taking absolute value") defocus1 = abs(defocus1) defocus2 = abs(defocus2) # get angle within range -90 < angle <= 90 while angle > 90: angle -= 180 while angle < -90: angle += 180 if debug is True: apDisplay.printColor( "Final params: def1: %.2e | def2: %.2e | angle: %.1f" % (defocus1, defocus2, angle), "cyan") perdiff = abs(defocus1 - defocus2) / abs(defocus1 + defocus2) print("Defocus Astig Percent Diff %.2f -- %.3e, %.3e" % (perdiff * 100, defocus1, defocus2)) ctfvalues['defocus1'] = defocus1 ctfvalues['defocus2'] = defocus2 ctfvalues['angle_astigmatism'] = angle return ctfvalues
def convertVolToSpider(self, mrcvolfile=None, modelid=None, apix=None, spivolfile=None): """ takes the mrc volume and creates a spider file ready for processing """ if modelid is not None: initModelData = appiondata.ApInitialModelData.direct_query(modelid) mrcvolfile = initModelData['path']['path'] + "/" + initModelData[ 'name'] apix = initModelData['pixelsize'] stackapix = apStack.getStackPixelSizeFromStackId( self.params['partstackid']) * self.params['bin'] stackboxsize = apStack.getStackBoxsize( self.params['partstackid']) / self.params['bin'] print apix print mrcvolfile if not os.path.isfile(mrcvolfile): apDisplay.printError("volfile does not exist: " + mrcvolfile) ### first high pass filter particles #apDisplay.printMsg("pre-filtering particles") #apix = apStack.getStackPixelSizeFromStackId(self.params['partstackid']) #emancmd = ("proc3d "+mrcvolfile+" "+emanstackfile # +" apix="+str(apix)+" hp="+str(self.params['highpasspart']) # +" inplace") #apEMAN.executeEmanCmd(emancmd, verbose=True) ### convert imagic stack to spider emancmd = "proc3d " emancmd += mrcvolfile + " " if spivolfile is None: spivolfile = os.path.join(self.params['rundir'], "threed-0a.spi") apFile.removeFile(spivolfile, warn=True) emancmd += spivolfile + " " emancmd += "scale=" + str(apix / stackapix) + " " emancmd += "clip=" + str(stackboxsize) + "," + str( stackboxsize) + "," + str(stackboxsize) + " " emancmd += "spidersingle" starttime = time.time() apDisplay.printColor("Running spider volume conversion", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor( "finished conversion in " + apDisplay.timeString(time.time() - starttime), "cyan") return spivolfile
def _startLoop(self, tiltseriesdata): """ initilizes several parameters for a new series and checks if it is okay to start processing series """ #calc series left self.stats['seriesleft'] = self.stats['seriescount'] - self.stats['count'] if self.params['background'] is False: apDisplay.printColor( "\nStarting series %d ( skip:%d, remain:%d ) id:%d" %(tiltseriesdata['number'], self.stats['skipcount'], self.stats['seriesleft'], tiltseriesdata.dbid,), "green") #only if a series was processed last if(self.stats['lastcount'] != self.stats['count']): sys.stderr.write("\n") self.stats['lastcount'] = self.stats['count'] self._checkMemLeak() # skip if last image belong to the series doesn't exist: imgtree = apDatabase.getImagesFromTiltSeries(tiltseriesdata,False) imgpath = os.path.join(tiltseriesdata['session']['image path'], imgtree[0]['filename']+'.mrc') if not os.path.isfile(imgpath): apDisplay.printWarning(imgpath+" not found, skipping") return False # skip if there are some problem with the series if self.__isShortTiltSeries(tiltseriesdata) or self.isBadTiltSeries(tiltseriesdata): apDisplay.printWarning("Series %d is not good enough for processing, skipping" % (tiltseriesdata['number'])) seriesname = "series%3d" % (tiltseriesdata['number']) self._writeDoneDict(seriesname) self.stats['count'] += 1 return False # check to see if series has already been processed if self._alreadyProcessed(tiltseriesdata): return False self.stats['waittime'] = 0 if self.reprocessSeries(tiltseriesdata) is True: if self.params['background'] is True: sys.stderr.write(",") else: """apDisplay.printMsg("reprocessing series %d" % (tiltseriesdata['number']))""" else: if self.params['background'] is True: sys.stderr.write(".") else: """apDisplay.printMsg("processing series %d" % (tiltseriesdata['number']))""" return True
def convertStackToSpider(self, emanstackfile): """ takes the stack file and creates a spider file ready for processing """ if not os.path.isfile(emanstackfile): apDisplay.printError("stackfile does not exist: " + emanstackfile) tempstack = os.path.join(self.params['rundir'], "filter" + self.timestamp + ".hed") ### first high pass filter particles apDisplay.printMsg("pre-filtering particles") apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid']) boxsize = self.getBoxSize() emancmd = ("proc2d " + emanstackfile + " " + tempstack + " apix=" + str(apix) + " ") if self.params['highpasspart'] is not None and self.params[ 'highpasspart'] > 0: emancmd += "hp=" + str(self.params['highpasspart']) + " " if self.params[ 'lowpasspart'] is not None and self.params['lowpasspart'] > 0: emancmd += "lp=" + str(self.params['lowpasspart']) + " " if self.params['tiltbin'] > 1: clipsize = boxsize * self.params['tiltbin'] emancmd += " shrink=%d clip=%d,%d " % (self.params['tiltbin'], clipsize, clipsize) apEMAN.executeEmanCmd(emancmd, verbose=True) ### convert imagic stack to spider emancmd = "proc2d " emancmd += tempstack + " " spiderstack = os.path.join(self.params['rundir'], "rctstack" + self.timestamp + ".spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack + " " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor( "Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) time.sleep(1) # wait a sec, for things to finish apDisplay.printColor( "finished eman in " + apDisplay.timeString(time.time() - starttime), "cyan") apFile.removeStack(tempstack, warn=False) apFile.removeStack(emanstackfile, warn=False) if not os.path.isfile(spiderstack): apDisplay.printError("Failed to create a spider stack") return spiderstack
def start(self): ### database entry parameters package_table = 'ApFrealignIterData|frealignParams' ### set projection-matching path self.projmatchpath = os.path.abspath( os.path.join(self.params['rundir'], "recon")) ### determine which iterations to upload lastiter = self.findLastCompletedIteration() uploadIterations = self.verifyUploadIterations(lastiter) for iteration in uploadIterations: apDisplay.printColor("uploading iteration %d" % iteration, "cyan") package_database_object = self.instantiateProjMatchParamsData( iteration) if not self.params['euleronly']: ### move FSC file to results directory self.FSCExists = self.convertFSCFileForIteration(iteration) ### create a text file with particle information self.createParticleDataFile(iteration, package_database_object) if not self.params['euleronly']: ### create mrc file of map for iteration and reference number oldvol = os.path.join(self.projmatchpath, "threed.%03da.mrc" % iteration) newvol = os.path.join( self.resultspath, "recon_%s_it%.3d_vol001.mrc" % (self.params['timestamp'], iteration)) apFile.safeCopy(oldvol, newvol) ### make chimera snapshot of volume self.createChimeraVolumeSnapshot(newvol, iteration) ### instantiate database objects self.insertRefinementRunData(iteration) self.insertRefinementIterationData(iteration, package_table, package_database_object) ### make symlink only after successful insertion if not self.params['euleronly'] and os.path.isfile(newvol): if os.path.isfile(oldvol): apFile.removeFile(oldvol, True) try: os.symlink(newvol, oldvol) except IOError, e: print e
def createJobFiles(self): self.goldStandardResolution() for i in range(self.params['numiter']): for side in ('left', 'right'): apDisplay.printColor("iteration %02d, side %s" % (i + 1, side), "cyan") stackFile = os.path.join("..", self.splitStacks[side]) oldprefix = "%s.iter%02d" % (side, i) newprefix = "%s.iter%02d" % (side, i + 1) self.createIterationJobRefineFiles(oldprefix, newprefix, stackFile) self.createReconstructionJobs(newprefix, stackFile)
def insertAlignParticlesIntoDatabase(self): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for ref in self.classD: ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = ref refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['iteration'] = 10 # if ref in self.resdict: # refq['ssnr_resolution'] = self.resdict[ref] ### setup particle info for partnum in self.classD[ref]: alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = int(partnum) alignpartq['alignstack'] = self.alignstackdata ### particle numbering starts with 1 in SIMPLE stackpartdata = apStack.getStackParticle( self.runparams['stackid'], int(partnum)) alignpartq['stackpart'] = stackpartdata alignpartq['ref'] = refq try: alignpartq['xshift'] = self.alignD[partnum - 1]['shx'] except: pass try: alignpartq['yshift'] = self.alignD[partnum - 1]['shy'] except: pass try: alignpartq['mirror'] = self.alignD[partnum - 1]['mirror'] except: pass alignpartq['rotation'] = self.alignD[partnum - 1]['rot'] ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(self.params['numpart']) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan") return
def insertClusterStack(self, classavg=None, classvar=None, numclass=None, insert=False): clusterstackq = appiondata.ApClusteringStackData() clusterstackq['avg_imagicfile'] = classavg + ".hed" clusterstackq['var_imagicfile'] = classvar + ".hed" clusterstackq['num_classes'] = numclass clusterstackq['clusterrun'] = self.clusterrun clusterstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) clusterstackq['hidden'] = False imagicfile = os.path.join(self.params['rundir'], clusterstackq['avg_imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find average stack file: " + imagicfile) imagicfile = os.path.join(self.params['rundir'], clusterstackq['var_imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find variance stack file: " + imagicfile) apDisplay.printMsg("inserting clustering stack into database") if insert is True: clusterstackq.insert() ### particle class data apDisplay.printColor( "Inserting particle classification data, please wait", "cyan") for i in range(numclass): classnum = i + 1 classdocfile = os.path.join( self.params['rundir'], "cluster/classdoc_%s_%04d.spi" % (self.timestamp, classnum)) partlist = self.readClassDocFile(classdocfile) sys.stderr.write(".") for partnum in partlist: alignpartdata = self.getAlignParticleData(partnum) cpartq = appiondata.ApClusteringParticleData() cpartq['clusterstack'] = clusterstackq cpartq['alignparticle'] = alignpartdata cpartq['partnum'] = partnum cpartq['refnum'] = classnum cpartq['clusterreference'] = None # actual parameters if insert is True: cpartq.insert() return
def setFileName(self, unique=False): if self.params['name'] is None: ### assign provided name if self.params['pdbid'] is not None: basename = "pdb%s-%s"%(self.params['pdbid'], self.timestamp) else: filebase = os.path.splitext(os.path.basename(self.params['pdbfile']))[0] basename = "%s-%s"%(filebase, self.timestamp) else: ### clean up provided name basename = os.path.splitext(os.path.basename(self.params['name']))[0] self.params['name'] = os.path.join(self.params['rundir'], basename) apDisplay.printColor("Naming PDB model: "+self.params['name'], "cyan") return