def setRunDir(self): """ this function only runs if no rundir is defined at the command line """ if self.params['rundir'] is None: if ('sessionname' in self.params and self.params['sessionname'] is not None ): # command line users may use sessionname rather than expId sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']]) else: if ('expId' in self.params and self.params['expId']): # expId should always be included from appionwrapper derived appionscript sessiondata = apDatabase.getSessionDataFromSessionId(self.params['expId']) self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']]) # The rest should not be needed with appionwrapper format from appionlib import apStack if ( self.params['rundir'] is None and 'reconid' in self.params and self.params['reconid'] is not None ): self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid'], msg=False) if ( self.params['rundir'] is None and 'stackid' in self.params and self.params['stackid'] is not None ): #auto set the run directory stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) path = os.path.abspath(stackdata['path']['path']) path = os.path.dirname(path) path = os.path.dirname(path) self.params['rundir'] = os.path.join(path, self.processdirname, self.params['runname']) self.params['outdir'] = self.params['rundir']
def checkConflicts(self): if self.params["reconid"] is None: apDisplay.printError("enter a reconstruction ID from the database") if self.params["mask"] is None: apDisplay.printError("enter a mask radius") if self.params["iter"] is None: apDisplay.printError("enter an iteration for the final Eulers") self.params["stackid"] = apStack.getStackIdFromRecon(self.params["reconid"]) self.params["sessionname"] = apStack.getSessionDataFromStackId(self.params["stackid"])
def checkConflicts(self): if self.params['reconid'] is None: apDisplay.printError("enter a reconstruction ID from the database") if self.params['mask'] is None: apDisplay.printError("enter a mask radius") if self.params['iter'] is None: apDisplay.printError("enter an iteration for the final Eulers") self.params['stackid'] = apStack.getStackIdFromRecon( self.params['reconid']) self.params['sessionname'] = apStack.getSessionDataFromStackId( self.params['stackid'])['name']
def getStackPartID(self, stackpartnum, reconrunid, stackid=None): if stackid is None: stackid = apStack.getStackIdFromRecon(reconrunid, msg=False) stackpartq = appiondata.ApStackParticleData() stackpartq['stack'] = appiondata.ApStackData.direct_query(stackid) stackpartq['particleNumber'] = stackpartnum stackpartdata = stackpartq.query(results=1) if not stackpartdata: apDisplay.printError("Failed to get Stack Particle ID for Number "+str(partnum)) return stackpartdata[0].dbid
def getStackPartID(self, stackpartnum, reconrunid, stackid=None): if stackid is None: stackid = apStack.getStackIdFromRecon(reconrunid, msg=False) stackpartq = appiondata.ApStackParticleData() stackpartq['stack'] = appiondata.ApStackData.direct_query(stackid) stackpartq['particleNumber'] = stackpartnum stackpartdata = stackpartq.query(results=1) if not stackpartdata: apDisplay.printError( "Failed to get Stack Particle ID for Number " + str(partnum)) return stackpartdata[0].dbid
def calculateEulerJumpsForEntireRecon(self, reconrunid, stackid=None, sym=None, multimodelrunid=None): if sym is None: sym = self.getSymmetry(reconrunid) if re.match("^icos", sym.lower()): apDisplay.printWarning("Processing Icos symmetry "+sym) if not re.match("^[cd][0-9]+$", sym.lower()) and not re.match("^icos", sym.lower()): apDisplay.printError("Cannot calculate euler jumps for symmetry: "+sym) return ### get stack particles if stackid is None: stackid = apStack.getStackIdFromRecon(reconrunid, msg=False) stackparts = apStack.getStackParticlesFromId(stackid) stackparts.sort(self.sortStackParts) numparts = len(stackparts) ### start loop t0 = time.time() medians = [] count = 0 miscount = 0 for stackpart in stackparts: count += 1 if multimodelrunid is None: jumpdata = self.getEulerJumpData(reconrunid, stackpartid=stackpart.dbid, stackid=stackid, sym=sym) else: jumpdata = self.getEulerJumpData(reconrunid, stackpartid=stackpart.dbid, stackid=stackid, sym=sym, multimodelrunid=multimodelrunid) if jumpdata is None: if miscount < 25: continue else: break medians.append(jumpdata['median']) if count % 500 == 0: timeremain = (time.time()-t0)/(count+1)*(numparts-count) print ("particle=% 5d; median jump=% 3.2f, remain time= %s" % (stackpart['particleNumber'], jumpdata['median'], apDisplay.timeString(timeremain))) if len(medians) > 0: ### print stats apDisplay.printMsg("complete "+str(len(stackparts))+" particles in "+apDisplay.timeString(time.time()-t0)) print "-- median euler jumper stats --" medians = numpy.asarray(medians, dtype=numpy.float32) print ("mean/std :: "+str(round(medians.mean(),2))+" +/- " +str(round(medians.std(),2))) print ("min/max :: "+str(round(medians.min(),2))+" <> " +str(round(medians.max(),2))) else: apDisplay.printWarning("no Euler jumpers inserted into the database, make sure that the angles are read by the recon uploader") return
def checkConflicts(self): ### check for missing and duplicate entries if self.params['iterid'] is None: apDisplay.printError("Please provide --iterid") if self.params['description'] is None: apDisplay.printError("Please provide --description") if self.params['runname'] is None: apDisplay.printError("Please provide --runname") ### get the stack ID from the other IDs self.iterdata = appiondata.ApRefineIterData.direct_query(self.params['iterid']) self.params['stackid'] = apStack.getStackIdFromRecon(self.iterdata['refineRun'].dbid) self.stackdata = apStack.getOnlyStackData(self.params['stackid']) ### check and make sure we got the stack id if self.params['stackid'] is None: apDisplay.printError("Could not find stackid from iterid")
def checkConflicts(self): """ make sure the necessary parameters are set correctly """ if self.cursor is None: # connect self.dbconf = sinedon.getConfig('appiondata') self.db = MySQLdb.connect(**self.dbconf) self.db.autocommit(True) # create a cursor self.cursor = self.db.cursor() if not self.params['reconid']: apDisplay.printError("Enter a Reconstruction Run ID, e.g. --reconid=243") if not self.params['tiltrunid']: self.params['tiltrunid'] = self.getTiltRunIDFromReconID(self.params['reconid']) self.params['symmetry'] = apSymmetry.getSymmetryFromReconRunId(self.params['reconid']) self.params['symmname'] = self.params['symmetry']['eman_name'] if not self.params['stackid']: self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid'])
def checkConflicts(self): if self.params['reconid'] is None: apDisplay.printError("enter a reconstruction ID from the database") if self.params['mask'] is None: apDisplay.printError("enter a mask radius") if self.params['iter'] is None: apDisplay.printError("enter an iteration for the final Eulers") if self.params['keeplist'] is None: apDisplay.printError("enter an keep list file") self.params['keeplist'] = os.path.abspath(self.params['keeplist']) if not os.path.isfile(self.params['keeplist']): apDisplay.printError("could not find list file") self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid']) if self.params['stackname'][-4:] != ".hed": s = os.path.splitext(self.params['stackname'])[0] s += ".hed" self.params['stackname'] = s apDisplay.printMsg("Stack name: "+self.params['stackname']) self.params['symmetry'] = apSymmetry.getSymmetryFromReconRunId(self.params['reconid']) self.params['symmname'] = self.params['symmetry']['eman_name']
def start(self): self.rootname = self.params['stackname'].split(".")[0] self.params['outputstack'] = os.path.join(self.params['rundir'], self.params['stackname']) if os.path.isfile(self.params['outputstack']): apFile.removeStack(self.params['outputstack']) if self.params['eotest'] is True: self.params['evenstack'] = os.path.splitext( self.params['outputstack'])[0] + '.even.hed' if os.path.isfile(self.params['evenstack']): apFile.removeStack(self.params['evenstack']) self.params['oddstack'] = os.path.splitext( self.params['outputstack'])[0] + '.odd.hed' if os.path.isfile(self.params['oddstack']): apFile.removeStack(self.params['oddstack']) classes = self.getClassData(self.params['reconid'], self.params['iter']) stackid = apStack.getStackIdFromRecon(self.params['reconid']) stackdata = apStack.getOnlyStackData(stackid) stackpath = os.path.join(stackdata['path']['path'], stackdata['name']) classkeys = classes.keys() classkeys.sort() classnum = 0 keeplist = self.procKeepList() finallist = [] apDisplay.printMsg("Processing " + str(len(classes)) + " classes") #loop through classes for key in classkeys: classnum += 1 if classnum % 10 == 1: sys.stderr.write("\b\b\b\b\b\b\b\b\b\b\b\b\b\b") sys.stderr.write( str(classnum) + " of " + (str(len(classkeys)))) # loop through particles in class classfile = self.rootname + "-class.lst" classf = open(classfile, 'w') classf.write('#LST\n') nptcls = 0 for ptcl in classes[key]['particles']: # translate DB into EMAN partnum = ptcl['particle']['particleNumber'] - 1 if partnum in keeplist: if ptcl['mirror']: mirror = 1 else: mirror = 0 rot = ptcl['euler3'] * math.pi / 180.0 classf.write("%d\t%s\t%f,\t%f,%f,%f,%d\n" % (partnum, stackpath, ptcl['quality_factor'], rot, ptcl['shiftx'], ptcl['shifty'], mirror)) nptcls += 1 finallist.append(partnum) classf.close() if nptcls < 1: continue self.makeClassAverages(classfile, self.params['outputstack'], classes[key], self.params['mask']) if self.params['eotest'] is True: self.makeEvenOddClasses(classfile, self.params['outputstack'], classes[key], self.params['mask']) apFile.removeFile(classfile) sys.stderr.write("\n") finalfilename = self.rootname + "-keep.lst" finalf = open(finalfilename, 'w') finallist.sort() for partnum in finallist: finalf.write('%d\n' % (partnum, )) finalf.close() stackstr = str(stackdata.dbid) reconstr = str(self.params['reconid']) ### recon 3d volumes threedname = os.path.join( self.params['rundir'], self.rootname + "." + str(self.params['iter']) + "a.mrc") emancmd = ("make3d " + self.params['outputstack'] + " out=" + threedname + " hard=50 sym=" + self.params['symmname'] + " pad=240 mask=" + str(self.params['mask']) + "; echo ''") #print emancmd apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True, logfile=self.rootname + "-eman.log") threednameb = os.path.join( self.params['rundir'], self.rootname + "." + str(self.params['iter']) + "b.mrc") emancmd = ("proc3d " + threedname + " " + threednameb + " apix=1.63 norm=0,1 lp=8 origin=0,0,0 mask=" + str(self.params['mask']) + "; echo '' ") apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True, logfile=self.rootname + "-eman.log") if self.params['eotest'] is True: # even evenname = os.path.join( self.params['rundir'], self.rootname + "-even." + str(self.params['iter']) + "a.mrc") if os.path.isfile(self.params['evenstack']): evenemancmd = ("make3d " + self.params['evenstack'] + " out=" + evenname + " hard=50 sym=" + self.params['symmname'] + " pad=240 mask=" + str(self.params['mask']) + "; echo ''") #print evenemancmd apEMAN.executeEmanCmd(evenemancmd, verbose=False, showcmd=True, logfile=self.rootname + "-eveneman.log") else: apDisplay.printWarning("file " + self.params['evenstack'] + " does not exist") # odd oddname = os.path.join( self.params['rundir'], self.rootname + "-odd." + str(self.params['iter']) + "a.mrc") if os.path.isfile(self.params['oddstack']): oddemancmd = ("make3d " + self.params['oddstack'] + " out=" + oddname + " hard=50 sym=" + self.params['symmname'] + " pad=240 mask=" + str(self.params['mask']) + "; echo ''") #print oddemancmd apEMAN.executeEmanCmd(oddemancmd, verbose=False, showcmd=True, logfile=self.rootname + "-oddeman.log") else: apDisplay.printWarning("file " + self.params['oddstack'] + " does not exist") #eotest fscout = os.path.join(self.params['rundir'], self.rootname + "-fsc.eotest") if os.path.isfile(oddname) and os.path.isfile(evenname): eotestcmd = "proc3d " + oddname + " " + evenname + " fsc=" + fscout apEMAN.executeEmanCmd(eotestcmd, verbose=True, showcmd=True) else: apDisplay.printWarning("could not perform eotest") if os.path.isfile(fscout): res = apRecon.getResolutionFromFSCFile(fscout, 160.0, 1.63) apDisplay.printColor(("resolution: %.5f" % (res)), "cyan") resfile = self.rootname + "-res.txt" f = open(resfile, 'a') f.write("[ %s ]\nresolution: %.5f\n" % (time.asctime(), res)) f.close()
def getSessionDataFromReconId(reconid): stackid = apStack.getStackIdFromRecon(reconid) partdata = apStack.getOneParticleFromStackId(stackid, msg=False) sessiondata = partdata['particle']['selectionrun']['session'] return sessiondata
def calculateEulerJumpsForEntireRecon(self, reconrunid, stackid=None, sym=None, multimodelrunid=None): if sym is None: sym = self.getSymmetry(reconrunid) if re.match("^icos", sym.lower()): apDisplay.printWarning("Processing Icos symmetry " + sym) if not re.match("^[cd][0-9]+$", sym.lower()) and not re.match( "^icos", sym.lower()): apDisplay.printError( "Cannot calculate euler jumps for symmetry: " + sym) return ### get stack particles if stackid is None: stackid = apStack.getStackIdFromRecon(reconrunid, msg=False) stackparts = apStack.getStackParticlesFromId(stackid) stackparts.sort(self.sortStackParts) numparts = len(stackparts) ### start loop t0 = time.time() medians = [] count = 0 miscount = 0 for stackpart in stackparts: count += 1 if multimodelrunid is None: jumpdata = self.getEulerJumpData(reconrunid, stackpartid=stackpart.dbid, stackid=stackid, sym=sym) else: jumpdata = self.getEulerJumpData( reconrunid, stackpartid=stackpart.dbid, stackid=stackid, sym=sym, multimodelrunid=multimodelrunid) if jumpdata is None: if miscount < 25: continue else: break medians.append(jumpdata['median']) if count % 500 == 0: timeremain = (time.time() - t0) / (count + 1) * (numparts - count) print("particle=% 5d; median jump=% 3.2f, remain time= %s" % (stackpart['particleNumber'], jumpdata['median'], apDisplay.timeString(timeremain))) if len(medians) > 0: ### print stats apDisplay.printMsg("complete " + str(len(stackparts)) + " particles in " + apDisplay.timeString(time.time() - t0)) print "-- median euler jumper stats --" medians = numpy.asarray(medians, dtype=numpy.float32) print("mean/std :: " + str(round(medians.mean(), 2)) + " +/- " + str(round(medians.std(), 2))) print("min/max :: " + str(round(medians.min(), 2)) + " <> " + str(round(medians.max(), 2))) else: apDisplay.printWarning( "no Euler jumpers inserted into the database, make sure that the angles are read by the recon uploader" ) return
def getSessionDataFromReconId(reconid): stackid = apStack.getStackIdFromRecon(reconid) partdata = apStack.getOneParticleFromStackId(stackid, msg=False) sessiondata = partdata["particle"]["selectionrun"]["session"] return sessiondata
def start(self): self.rootname = self.params['stackname'].split(".")[0] self.params['outputstack'] = os.path.join(self.params['rundir'], self.params['stackname']) if os.path.isfile(self.params['outputstack']): apFile.removeStack(self.params['outputstack']) if self.params['eotest'] is True: self.params['evenstack'] = os.path.splitext(self.params['outputstack'])[0]+'.even.hed' if os.path.isfile(self.params['evenstack']): apFile.removeStack(self.params['evenstack']) self.params['oddstack'] = os.path.splitext(self.params['outputstack'])[0]+'.odd.hed' if os.path.isfile(self.params['oddstack']): apFile.removeStack(self.params['oddstack']) classes = self.getClassData(self.params['reconid'], self.params['iter']) stackid = apStack.getStackIdFromRecon(self.params['reconid']) stackdata = apStack.getOnlyStackData(stackid) stackpath = os.path.join(stackdata['path']['path'], stackdata['name']) classkeys = classes.keys() classkeys.sort() classnum=0 keeplist = self.procKeepList() finallist = [] apDisplay.printMsg("Processing "+str(len(classes))+" classes") #loop through classes for key in classkeys: classnum+=1 if classnum%10 == 1: sys.stderr.write("\b\b\b\b\b\b\b\b\b\b\b\b\b\b") sys.stderr.write(str(classnum)+" of "+(str(len(classkeys)))) # loop through particles in class classfile = self.rootname+"-class.lst" classf = open(classfile, 'w') classf.write('#LST\n') nptcls=0 for ptcl in classes[key]['particles']: # translate DB into EMAN partnum = ptcl['particle']['particleNumber'] - 1 if partnum in keeplist: if ptcl['mirror']: mirror=1 else: mirror=0 rot = ptcl['euler3']*math.pi/180.0 classf.write( "%d\t%s\t%f,\t%f,%f,%f,%d\n" % (partnum, stackpath, ptcl['quality_factor'], rot, ptcl['shiftx'], ptcl['shifty'], mirror)) nptcls+=1 finallist.append(partnum) classf.close() if nptcls<1: continue self.makeClassAverages(classfile, self.params['outputstack'], classes[key], self.params['mask']) if self.params['eotest'] is True: self.makeEvenOddClasses(classfile, self.params['outputstack'], classes[key], self.params['mask']) apFile.removeFile(classfile) sys.stderr.write("\n") finalfilename = self.rootname+"-keep.lst" finalf = open(finalfilename, 'w') finallist.sort() for partnum in finallist: finalf.write('%d\n' % (partnum,) ) finalf.close() stackstr = str(stackdata.dbid) reconstr = str(self.params['reconid']) ### recon 3d volumes threedname = os.path.join(self.params['rundir'], self.rootname+"."+str(self.params['iter'])+"a.mrc") emancmd = ( "make3d "+self.params['outputstack']+" out=" +threedname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" ) #print emancmd apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eman.log") threednameb = os.path.join(self.params['rundir'], self.rootname+"."+str(self.params['iter'])+"b.mrc") emancmd = ( "proc3d "+threedname+" "+threednameb +" apix=1.63 norm=0,1 lp=8 origin=0,0,0 mask="+str(self.params['mask'])+"; echo '' " ) apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eman.log") if self.params['eotest'] is True: # even evenname = os.path.join(self.params['rundir'], self.rootname+"-even."+str(self.params['iter'])+"a.mrc") if os.path.isfile(self.params['evenstack']): evenemancmd = ( "make3d "+self.params['evenstack']+" out=" +evenname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" ) #print evenemancmd apEMAN.executeEmanCmd(evenemancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eveneman.log") else: apDisplay.printWarning("file "+self.params['evenstack']+" does not exist") # odd oddname = os.path.join(self.params['rundir'], self.rootname+"-odd."+str(self.params['iter'])+"a.mrc") if os.path.isfile(self.params['oddstack']): oddemancmd = ( "make3d "+self.params['oddstack']+" out=" +oddname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" ) #print oddemancmd apEMAN.executeEmanCmd(oddemancmd, verbose=False, showcmd=True, logfile=self.rootname+"-oddeman.log") else: apDisplay.printWarning("file "+self.params['oddstack']+" does not exist") #eotest fscout = os.path.join(self.params['rundir'], self.rootname+"-fsc.eotest") if os.path.isfile(oddname) and os.path.isfile(evenname): eotestcmd = "proc3d "+oddname+" "+evenname+" fsc="+fscout apEMAN.executeEmanCmd(eotestcmd, verbose=True, showcmd=True) else: apDisplay.printWarning("could not perform eotest") if os.path.isfile(fscout): res = apRecon.getResolutionFromFSCFile(fscout, 160.0, 1.63) apDisplay.printColor( ("resolution: %.5f" % (res)), "cyan") resfile = self.rootname+"-res.txt" f = open(resfile, 'a') f.write("[ %s ]\nresolution: %.5f\n" % (time.asctime(), res)) f.close()