def medianVolume(self): volpath = os.path.join(self.params['rundir'], "volumes/*a.mrc") mrcfiles = glob.glob(volpath) volumes = [] for filename in mrcfiles: if os.path.isfile(filename): vol = mrc.read(filename) print filename, vol.shape volumes.append(vol) volarray = numpy.asarray(volumes, dtype=numpy.float32) try: medarray = numpy.median(volarray, axis=0) except: medarray = numpy.median(volarray) medfile = os.path.join(self.params['rundir'], "volumes/medianVolume.mrc") print medfile, medarray.shape mrc.write(medarray, medfile) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) uploadcmd = ( ("uploadModel.py --projectid=%d --session=%s --file=%s " +"--apix=%.3f --sym=%s --name=satmedian-recon%d.mrc --res=30 --description='%s %d'") %(self.params['projectid'], sessiondata['name'], medfile, apix, self.params['symmname'], self.params['reconid'], "SAT selected median volume for recon", self.params['reconid'], ) ) apDisplay.printColor(uploadcmd, "purple") f = open("upload.sh", "w") f.write(uploadcmd+"\n") f.close()
def useClusterForTemplate(self): apDisplay.printMsg("Using clustering stack to make templates") clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid']) self.params['apix'] = clusterstackdata['clusterrun']['alignstack']['pixelsize'] stackfile = os.path.join(clusterstackdata['path']['path'], clusterstackdata['avg_imagicfile']) self.params['templatelist'] = [] stackid = clusterstackdata['clusterrun']['alignstack']['stack'].dbid sessiondata = apStack.getSessionDataFromStackId(stackid) self.params['sessionname'] = sessiondata['name'] ### check to see if stackimagenum is within the boundary of the stack numpart = apFile.numImagesInStack(stackfile) for i in self.params['imgnums'].split(","): partnum = int(i) if partnum > numpart or partnum < 0: apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart)) apDisplay.printMsg("Extracting image %d from cluster stack" % partnum) templatename = "cluster%d-average%d.mrc"%(self.params['clusterid'],partnum) abstemplatepath= os.path.join(clusterstackdata['path']['path'], templatename) ### run proc2d with params['stackimgnum'] emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum) ### create template apDisplay.printMsg("creating "+templatename) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'].append(abstemplatepath)
def checkConflicts(self): ### setup correct database after we have read the project id if 'projectid' in self.params and self.params['projectid'] is not None: apDisplay.printMsg("Using split database") # use a project database newdbname = apProject.getAppionDBFromProjectId( self.params['projectid']) sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '" + newdbname + "'", "green") # DD processes self.dd = apDDprocess.DDStackProcessing() print self.dd # get stack data self.stackdata = appiondata.ApStackData.direct_query( self.params['stackid']) self.stackparts = apStack.getStackParticlesFromId( self.params['stackid'], msg=True) self.sessiondata = apStack.getSessionDataFromStackId( self.params['stackid']) # query image qimage = self.stackparts[0]['particle']['image'] # DD info self.dd.setImageData(qimage) self.dd.setDDStackRun(self.params['ddstackid']) self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def checkConflicts(self): ### setup correct database after we have read the project id if 'projectid' in self.params and self.params['projectid'] is not None: apDisplay.printMsg("Using split database") # use a project database newdbname = apProject.getAppionDBFromProjectId(self.params['projectid']) sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '"+newdbname+"'", "green") # DD processes self.dd = apDDprocess.DDStackProcessing() print self.dd # get stack data self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid']) self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True) self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) # query image qimage = self.stackparts[0]['particle']['image'] # DD info self.dd.setImageData(qimage) self.dd.setDDStackRun(self.params['ddstackid']) self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def getFormattedStackInfo(stackdata, format): sessiondata = apStack.getSessionDataFromStackId(stackdata.dbid) stackroot = os.path.splitext(stackdata["name"])[0] instackdir = stackdata["path"]["path"] outstackdir = os.path.join( appionbasepath, sessiondata["name"], "stacks", os.path.basename(stackdata["path"]["path"]) + "_" + format ) return stackroot, instackdir, outstackdir
def getFormattedStackInfo(stackdata, format): sessiondata = apStack.getSessionDataFromStackId(stackdata.dbid) stackroot = os.path.splitext(stackdata['name'])[0] instackdir = stackdata['path']['path'] outstackdir = os.path.join( appionbasepath, sessiondata['name'], 'stacks', os.path.basename(stackdata['path']['path']) + '_' + format) return stackroot, instackdir, outstackdir
def insertOtrRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq["type"] = "fsc" fscresq["half"] = self.fscresolution fscresq["fscfile"] = "fscdata" + self.timestamp + ".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq["type"] = "rmeasure" rmeasureq["half"] = self.rmeasureresolution rmeasureq["fscfile"] = None ### insert rct run data otrrunq = appiondata.ApOtrRunData() otrrunq["runname"] = self.params["runname"] tempstr = "" for cnum in self.classlist: tempstr += str(cnum) + "," classliststr = tempstr[:-1] otrrunq["classnums"] = classliststr otrrunq["numiter"] = self.params["numiters"] otrrunq["euleriter"] = self.params["refineiters"] otrrunq["maskrad"] = self.params["radius"] otrrunq["lowpassvol"] = self.params["lowpassvol"] otrrunq["highpasspart"] = self.params["highpasspart"] otrrunq["median"] = self.params["median"] otrrunq["description"] = self.params["description"] otrrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"])) otrrunq["alignstack"] = self.alignstackdata otrrunq["tiltstack"] = apStack.getOnlyStackData(self.params["tiltstackid"]) otrrunq["numpart"] = self.numpart otrrunq["fsc_resolution"] = fscresq otrrunq["rmeasure_resolution"] = rmeasureq if self.params["commit"] is True: otrrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq["otrrun"] = otrrunq densq["path"] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq["name"] = os.path.basename(volfile) densq["hidden"] = False densq["norm"] = True densq["symmetry"] = appiondata.ApSymmetryData.direct_query(25) densq["pixelsize"] = apStack.getStackPixelSizeFromStackId(self.params["partstackid"]) * self.params["bin"] densq["boxsize"] = self.getBoxSize() densq["lowpass"] = self.params["lowpassvol"] densq["highpass"] = self.params["highpasspart"] densq["mask"] = self.params["radius"] # densq['iterid'] = self.params['numiters'] densq["description"] = self.params["description"] densq["resolution"] = self.fscresolution densq["rmeasure"] = self.rmeasureresolution densq["session"] = apStack.getSessionDataFromStackId(self.params["tiltstackid"]) densq["md5sum"] = apFile.md5sumfile(volfile) if self.params["commit"] is True: densq.insert() return
def insertOtrRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq['type'] = "fsc" fscresq['half'] = self.fscresolution fscresq['fscfile'] = "fscdata"+self.timestamp+".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq['type'] = "rmeasure" rmeasureq['half'] = self.rmeasureresolution rmeasureq['fscfile'] = None ### insert rct run data otrrunq = appiondata.ApOtrRunData() otrrunq['runname'] = self.params['runname'] tempstr = "" for cnum in self.classlist: tempstr += str(cnum)+"," classliststr = tempstr[:-1] otrrunq['classnums'] = classliststr otrrunq['numiter'] = self.params['numiters'] otrrunq['euleriter'] = self.params['refineiters'] otrrunq['maskrad'] = self.params['radius'] otrrunq['lowpassvol'] = self.params['lowpassvol'] otrrunq['highpasspart'] = self.params['highpasspart'] otrrunq['median'] = self.params['median'] otrrunq['description'] = self.params['description'] otrrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) otrrunq['alignstack'] = self.alignstackdata otrrunq['tiltstack'] = apStack.getOnlyStackData(self.params['tiltstackid']) otrrunq['numpart'] = self.numpart otrrunq['fsc_resolution'] = fscresq otrrunq['rmeasure_resolution'] = rmeasureq if self.params['commit'] is True: otrrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['otrrun'] = otrrunq densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25) densq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] densq['boxsize'] = self.getBoxSize() densq['lowpass'] = self.params['lowpassvol'] densq['highpass'] = self.params['highpasspart'] densq['mask'] = self.params['radius'] #densq['iterid'] = self.params['numiters'] densq['description'] = self.params['description'] densq['resolution'] = self.fscresolution densq['rmeasure'] = self.rmeasureresolution densq['session'] = apStack.getSessionDataFromStackId(self.params['tiltstackid']) densq['md5sum'] = apFile.md5sumfile(volfile) if self.params['commit'] is True: densq.insert() return
def insertRefinementRunData(self, iteration, reference_number=1): ''' fills in database entry object for ApRefineRunData table ''' ### fill in ApMultiModelRefineRunData object, if valid if self.multiModelRefinementRun is True: multimodelq = appiondata.ApMultiModelRefineRunData() multimodelq['runname'] = self.params['runname'] multimodelq[ 'REF|projectdata|projects|project'] = apProject.getProjectIdFromStackId( self.runparams['stackid']) multimodelq['session'] = apStack.getSessionDataFromStackId( self.runparams['stackid']) multimodelq['num_refinements'] = self.runparams[ 'NumberOfReferences'] self.multimodelq = multimodelq ### fill in ApRefineRunData object runq = appiondata.ApRefineRunData() runq['runname'] = self.params['runname'] runq['stack'] = self.stackdata runq['reference_number'] = reference_number earlyresult = runq.query(results=1) # check unique run if earlyresult: apDisplay.printWarning( "Run already exists in the database.\nIdentical data will not be reinserted" ) paramdescription = self.params['description'] if not paramdescription: paramdescription = None runq['job'] = self.params['jobinfo'] if len(self.modeldata) > 1: runq['initialModel'] = self.modeldata[reference_number - 1] else: runq['initialModel'] = self.modeldata[0] runq['package'] = self.package runq['path'] = appiondata.ApPathData( path=os.path.abspath(self.resultspath)) runq['description'] = paramdescription runq['num_iter'] = self.runparams['numiter'] if self.multiModelRefinementRun is True: runq['multiModelRefineRun'] = self.multimodelq result = runq.query(results=1) # save run entry in the parameters if result: self.refinerunq = result[0] elif self.params['commit'] is True: apDisplay.printMsg( "Refinement Run was not found, setting to inserted values") self.refinerunq = runq else: apDisplay.printWarning( "Refinement Run was not found, setting to 'None'") self.refinerunq = None return
def checkConflicts(self): if self.params["reconid"] is None: apDisplay.printError("enter a reconstruction ID from the database") if self.params["mask"] is None: apDisplay.printError("enter a mask radius") if self.params["iter"] is None: apDisplay.printError("enter an iteration for the final Eulers") self.params["stackid"] = apStack.getStackIdFromRecon(self.params["reconid"]) self.params["sessionname"] = apStack.getSessionDataFromStackId(self.params["stackid"])
def checkConflicts(self): ### make sure the necessary parameters are set if self.params['description'] is None: apDisplay.printError("enter a template description") if self.params['diam'] is None: apDisplay.printError("enter the particle diameter in Angstroms") ### make sure we have something if (self.params['template'] is None and self.params['stackid'] is None and self.params['clusterid'] is None and self.params['alignid'] is None ): apDisplay.printError("enter a template root name (wild cards are valid)") ### check if apix is needed if (self.params['apix'] is None and self.params['stackid'] is None and self.params['clusterid'] is None and self.params['alignid'] is None ): apDisplay.printError("enter a pixel size") ### check for session if self.params['sessionname'] is None: if self.params['alignid'] is not None: alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignid']) stackid = alignstackdata['stack'].dbid sessiondata = apStack.getSessionDataFromStackId(stackid) self.params['sessionname'] = sessiondata['name'] elif self.params['stackid'] is not None: stackid = self.params['stackid'] sessiondata = apStack.getSessionDataFromStackId(stackid) self.params['sessionname'] = sessiondata['name'] elif self.params['alignid'] is not None: clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid']) stackid = clusterstackdata['clusterrun']['alignstack']['stack'].dbid sessiondata = apStack.getSessionDataFromStackId(stackid) self.params['sessionname'] = sessiondata['name'] if self.params['sessionname'] is None: apDisplay.printError("Could not find session") if self.params['template'] is not None: self.params['template'] = os.path.abspath(self.params['template'])
def checkConflicts(self): if self.params['reconid'] is None: apDisplay.printError("enter a reconstruction ID from the database") if self.params['mask'] is None: apDisplay.printError("enter a mask radius") if self.params['iter'] is None: apDisplay.printError("enter an iteration for the final Eulers") self.params['stackid'] = apStack.getStackIdFromRecon( self.params['reconid']) self.params['sessionname'] = apStack.getSessionDataFromStackId( self.params['stackid'])['name']
def start(self): ### check for existing run selectrunq = appiondata.ApSelectionRunData() selectrunq['name'] = self.params['runname'] selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) selectrundata = selectrunq.query(readimages=False) if selectrundata: apDisplay.printError("Runname already exists") ### stack data stackdata = apStack.getOnlyStackData(self.params['stackid']) ### stack particles stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True) stackparts.reverse() ### selection run for first particle oldselectrun = stackparts[0]['particle']['selectionrun'] ### set selection run manualparamsq = appiondata.ApManualParamsData() manualparamsq['diam'] = self.getDiamFromSelectionRun(oldselectrun) manualparamsq['oldselectionrun'] = oldselectrun manualparamsq['trace'] = False selectrunq = appiondata.ApSelectionRunData() selectrunq['name'] = self.params['runname'] selectrunq['hidden'] = False selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) selectrunq['session'] = apStack.getSessionDataFromStackId(self.params['stackid']) selectrunq['manparams'] = manualparamsq ### insert particles apDisplay.printMsg("Inserting particles into database") count = 0 t0 = time.time() startmem = mem.active() numpart = len(stackparts) for stackpart in stackparts: count += 1 if count > 10 and count%100 == 0: perpart = (time.time()-t0)/float(count+1) apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"% (count, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart), apDisplay.timeString(perpart*(numpart-count))), "blue") oldpartdata = stackpart['particle'] newpartq = appiondata.ApParticleData(initializer=oldpartdata) newpartq['selectionrun'] = selectrunq if self.params['commit'] is True: newpartq.insert() apDisplay.printMsg("Completed in %s"%(apDisplay.timeString(time.time()-t0)))
def getSessionData(self): sessiondata = None if 'sessionname' in self.params and self.params['sessionname'] is not None: sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) if not sessiondata and 'stackid' in self.params: from appionlib import apStack sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) if not sessiondata: ### works with only canonical session names s = re.search('/([0-9][0-9][a-z][a-z][a-z][0-9][0-9][^/]*)/', self.params['rundir']) if s: self.params['sessionname'] = s.groups()[0] sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) return sessiondata
def insertRefinementRunData(self, iteration, reference_number=1): """ fills in database entry object for ApRefineRunData table """ ### fill in ApMultiModelRefineRunData object, if valid if self.multiModelRefinementRun is True: multimodelq = appiondata.ApMultiModelRefineRunData() multimodelq["runname"] = self.params["runname"] multimodelq["REF|projectdata|projects|project"] = apProject.getProjectIdFromStackId( self.runparams["stackid"] ) multimodelq["session"] = apStack.getSessionDataFromStackId(self.runparams["stackid"]) multimodelq["num_refinements"] = self.runparams["NumberOfReferences"] self.multimodelq = multimodelq ### fill in ApRefineRunData object runq = appiondata.ApRefineRunData() runq["runname"] = self.params["runname"] runq["stack"] = self.stackdata runq["reference_number"] = reference_number earlyresult = runq.query(results=1) # check unique run if earlyresult: apDisplay.printWarning("Run already exists in the database.\nIdentical data will not be reinserted") paramdescription = self.params["description"] if not paramdescription: paramdescription = None runq["job"] = self.params["jobinfo"] if len(self.modeldata) > 1: runq["initialModel"] = self.modeldata[reference_number - 1] else: runq["initialModel"] = self.modeldata[0] runq["package"] = self.package runq["path"] = appiondata.ApPathData(path=os.path.abspath(self.resultspath)) runq["description"] = paramdescription runq["num_iter"] = self.runparams["numiter"] if self.multiModelRefinementRun is True: runq["multiModelRefineRun"] = self.multimodelq result = runq.query(results=1) # save run entry in the parameters if result: self.refinerunq = result[0] elif self.params["commit"] is True: apDisplay.printMsg("Refinement Run was not found, setting to inserted values") self.refinerunq = runq else: apDisplay.printWarning("Refinement Run was not found, setting to 'None'") self.refinerunq = None return
def useStackForTemplate(self): apDisplay.printMsg("Using stack to make templates") sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) self.params['sessionname'] = sessiondata['name'] self.params['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) stackdata = apStack.getOnlyStackData(self.params['stackid']) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) self.params['templatelist'] = [] if self.params['avgstack'] is True: ### averaging all particles together apDisplay.printMsg("Averaging all images in stack") templatename = "stack%d-average.mrc" % (self.params['stackid']) oldtemplatepath = os.path.join(stackdata['path']['path'], "average.mrc") abstemplatepath = os.path.join(stackdata['path']['path'], templatename) if os.path.isfile(oldtemplatepath): shutil.copy(oldtemplatepath, abstemplatepath) else: ### average all images using proc2d emancmd = "proc2d %s %s average" % (stackfile, abstemplatepath) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'] = [abstemplatepath] elif self.params['imgnums'] is not None: ### check to see if stackimagenum is within the boundary of the stack numpart = apFile.numImagesInStack(stackfile) for i in self.params['imgnums'].split(","): partnum = int(i) if partnum > numpart or partnum < 0: apDisplay.printError( "'imgnums' is NOT within the boundary of the stack: %d > %d" % (partnum, numpart)) apDisplay.printMsg("Extracting image %d from stack" % partnum) templatename = "stack%d-particle%d.mrc" % ( self.params['stackid'], partnum) abstemplatepath = os.path.join(self.params['rundir'], templatename) ### run proc2d with params['stackimgnum'] emancmd = "proc2d %s %s first=%d last=%d" % ( stackfile, abstemplatepath, partnum, partnum) ### create template apDisplay.printMsg("creating " + templatename) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'].append(abstemplatepath)
def getSessionData(self): sessiondata = None if "sessionname" in self.params and self.params["sessionname"] is not None: sessiondata = apDatabase.getSessionDataFromSessionName(self.params["sessionname"]) if not sessiondata and "stackid" in self.params: from appionlib import apStack sessiondata = apStack.getSessionDataFromStackId(self.params["stackid"]) if not sessiondata: ### works with only canonical session names s = re.search("/([0-9][0-9][a-z][a-z][a-z][0-9][0-9][^/]*)/", self.params["rundir"]) if s: self.params["sessionname"] = s.groups()[0] sessiondata = apDatabase.getSessionDataFromSessionName(self.params["sessionname"]) return sessiondata
def useStackForTemplate(self): apDisplay.printMsg("Using stack to make templates") sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) self.params['sessionname'] = sessiondata['name'] self.params['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) stackdata = apStack.getOnlyStackData(self.params['stackid']) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) self.params['templatelist'] = [] if self.params['avgstack'] is True: ### averaging all particles together apDisplay.printMsg("Averaging all images in stack") templatename = "stack%d-average.mrc"%(self.params['stackid']) oldtemplatepath = os.path.join(stackdata['path']['path'], "average.mrc") abstemplatepath = os.path.join(stackdata['path']['path'], templatename) if os.path.isfile(oldtemplatepath): shutil.copy(oldtemplatepath, abstemplatepath) else: ### average all images using proc2d emancmd = "proc2d %s %s average" % (stackfile, abstemplatepath) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'] = [abstemplatepath] elif self.params['imgnums'] is not None: ### check to see if stackimagenum is within the boundary of the stack numpart = apFile.numImagesInStack(stackfile) for i in self.params['imgnums'].split(","): partnum = int(i) if partnum > numpart or partnum < 0: apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart)) apDisplay.printMsg("Extracting image %d from stack" % partnum) templatename = "stack%d-particle%d.mrc"%(self.params['stackid'],partnum) abstemplatepath= os.path.join(self.params['rundir'], templatename) ### run proc2d with params['stackimgnum'] emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum) ### create template apDisplay.printMsg("creating "+templatename) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'].append(abstemplatepath)
def uploadDensity(self, volfile): ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = self.symmdata densq['pixelsize'] = self.clusterstackdata['clusterrun']['pixelsize'] densq['boxsize'] = self.clusterstackdata['clusterrun']['boxsize'] #densq['lowpass'] = None #densq['highpass'] = None densq['mask'] = self.params['mask'] densq['description'] = self.params['description']+"- From eman start-"+self.params['method'] #densq['resolution'] = None densq['session'] = apStack.getSessionDataFromStackId(self.params['stackid']) densq['md5sum'] = apFile.md5sumfile(volfile) densq['eman'] = self.params['method'] if self.params['commit'] is True: densq.insert() return
def start(self): ### check for existing run selectrunq = appiondata.ApSelectionRunData() selectrunq["name"] = self.params["runname"] selectrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"])) selectrundata = selectrunq.query(readimages=False) if selectrundata: apDisplay.printError("Runname already exists") if self.params["ddstack"]: self.other_ddstack_used = [] self.dd = apDDprocess.DDStackProcessing() self.dd.setDDStackRun(self.params["ddstack"]) self.newddstackrun = self.dd.getDDStackRun(show_msg=True) ### stack data stackdata = apStack.getOnlyStackData(self.params["stackid"]) ### stack particles stackparts = apStack.getStackParticlesFromId(self.params["stackid"], msg=True) stackparts.reverse() ### selection run for first particle oldselectrun = stackparts[0]["particle"]["selectionrun"] ### set selection run manualparamsq = appiondata.ApManualParamsData() manualparamsq["diam"] = self.getDiamFromSelectionRun(oldselectrun) manualparamsq["oldselectionrun"] = oldselectrun manualparamsq["trace"] = False selectrunq = appiondata.ApSelectionRunData() selectrunq["name"] = self.params["runname"] selectrunq["hidden"] = False selectrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"])) selectrunq["session"] = apStack.getSessionDataFromStackId(self.params["stackid"]) selectrunq["manparams"] = manualparamsq ### insert particles apDisplay.printMsg("Inserting particles into database") count = 0 t0 = time.time() startmem = mem.active() numpart = len(stackparts) for stackpart in stackparts: count += 1 if count > 10 and count % 100 == 0: perpart = (time.time() - t0) / float(count + 1) apDisplay.printColor( "part %d of %d :: %.1fM mem :: %s/part :: %s remain" % ( count, numpart, (mem.active() - startmem) / 1024.0, apDisplay.timeString(perpart), apDisplay.timeString(perpart * (numpart - count)), ), "blue", ) oldpartdata = stackpart["particle"] newpartq = appiondata.ApParticleData(initializer=oldpartdata) newpartq["selectionrun"] = selectrunq if self.params["ddstack"]: newimagedata = self.getNewImageFromDDStack(oldpartdata["image"]) if newimagedata is False: # no pick transferred continue newpartq["image"] = newimagedata if self.params["commit"] is True: newpartq.insert() apDisplay.printMsg("Completed in %s" % (apDisplay.timeString(time.time() - t0)))
def checkConflicts(self): ### setup correct database after we have read the project id if 'projectid' in self.params and self.params['projectid'] is not None: apDisplay.printMsg("Using split database") # use a project database newdbname = apProject.getAppionDBFromProjectId( self.params['projectid']) sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '" + newdbname + "'", "green") if self.params['stackid'] is None: apDisplay.printError("stackid was not defined") if self.params['expweight'] is False: apDisplay.printWarning( "Exposure weighting is turned off, make sure this is what you want" ) if self.params['localavg'] is False: apDisplay.printWarning( "Trajectory local averaging is turned off, make sure this is what you want" ) # DD processes self.dd = apDDprocess.DDStackProcessing() print self.dd # get stack data self.stackdata = appiondata.ApStackData.direct_query( self.params['stackid']) self.stackparts = apStack.getStackParticlesFromId( self.params['stackid'], msg=True) self.sessiondata = apStack.getSessionDataFromStackId( self.params['stackid']) # query image qimage = self.stackparts[0]['particle']['image'] # pixel size info self.params['apix'] = apStack.getMicrographPixelSizeFromStackId( self.params['stackid']) self.params['box'] = self.stackdata['boxsize'] self.params['particleradius'] = self.params[ 'particleradius'] / self.params['apix'] if self.params['particleradius'] > self.params['box'] / 2.0: apDisplay.printWarning( "specified particle radius greater than box radius, \ setting particle radius to 0.8 * boxsize") # micrograph & frame info frames = qimage['use frames'] nframes = len(frames) if self.params['framelastali'] is None: self.params['framelastali'] = frames[-1] if self.params['framelastave'] is None: self.params['framelastave'] = frames[-1] # microscope kV self.params['kv'] = qimage['scope']['high tension'] / 1000.0 # query exposure per frame, if not set here if self.params['total_dose'] is not None: dose = self.params['total_dose'] else: try: dose = apDatabase.getDoseFromImageData(qimage) except: apDisplay.printError( "dose not specified and not in database, please specify explicitly" ) if self.params['expperframe'] is None and self.params[ 'expweight'] is True: if dose is not None: self.params['expperframe'] = dose / nframes else: apDisplay.printError( "exposure per frame needs to be specified, cannot find in database" ) # dimensions self.params['framex'] = int( apDatabase.getDimensionsFromImageData(qimage)['x']) self.params['framey'] = int( apDatabase.getDimensionsFromImageData(qimage)['y']) # DD info self.dd.setImageData(qimage) self.dd.setDDStackRun(self.params['ddstackid']) self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def getFormattedStackInfo(stackdata,format): sessiondata = apStack.getSessionDataFromStackId(stackdata.dbid) stackroot = os.path.splitext(stackdata['name'])[0] instackdir = stackdata['path']['path'] outstackdir = os.path.join(appionbasepath,sessiondata['name'],'stacks',os.path.basename(stackdata['path']['path'])+'_'+format) return stackroot,instackdir,outstackdir
def start(self): self.params['outputstack'] = os.path.join(self.params['rundir'], self.params['stackname']) particles, self.params['refineiter'] = getParticleInfo( self.params['reconid'], self.params['iter']) stackdata = particles[0]['particle']['stack'] stack = os.path.join(stackdata['path']['path'], stackdata['name']) classes, cstats = determineClasses(particles) rejectlst = [] if self.params['sigma'] is not None: cutoff = cstats[ 'meanquality'] + self.params['sigma'] * cstats['stdquality'] apDisplay.printMsg("Cutoff = " + str(cutoff)) rejectlst = self.removePtclsByQualityFactor( particles, rejectlst, cutoff) if self.params['avgjump'] is not None: rejectlst = self.removePtclsByJumps(particles, rejectlst) if self.params['rejectlst']: rejectlst = removePtclsByLst(rejectlst, self.params) classkeys = classes.keys() classkeys.sort() classnum = 0 totalptcls = 0 keepfile = open('keep.lst', 'w') keepfile.write('#LST\n') reject = open('reject.lst', 'w') reject.write('#LST\n') apDisplay.printMsg("Processing classes") #loop through classes for key in classkeys: # file to hold particles of this class clsfile = open('clstmp.lst', 'w') clsfile.write('#LST\n') classnum += 1 if classnum % 10 == 1: apDisplay.printMsg( str(classnum) + " of " + (str(len(classkeys)))) images = EMAN.EMData() #loop through particles in class nptcls = 0 for ptcl in classes[key]['particles']: if ptcl['mirror']: mirror = 1 else: mirror = 0 rot = ptcl['euler3'] rot = rot * math.pi / 180 if ptcl['particle']['particleNumber'] not in rejectlst: l = '%d\t%s\t%f,\t%f,%f,%f,%d\n' % ( ptcl['particle']['particleNumber'] - 1, stack, ptcl['quality_factor'], rot, ptcl['shiftx'], ptcl['shifty'], mirror) keepfile.write(l) clsfile.write(l) totalptcls += 1 nptcls += 1 else: reject.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber'] - 1, stack, ptcl['quality_factor'], rot, ptcl['shiftx'], ptcl['shifty'], mirror)) #if ptcl['quality_factor']>cstats['meanquality']+3*cstats['stdquality']: # high.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1, # stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror)) clsfile.close() if nptcls < 1: continue if self.params['skipavg'] is False: makeClassAverages('clstmp.lst', self.params['outputstack'], classes[key], self.params) if self.params['eotest'] is True: self.makeEvenOddClasses('clstmp.lst', classes[key]) apDisplay.printMsg("\n") reject.close() keepfile.close() os.remove('clstmp.lst') # make 3d density file if specified: if self.params['make3d'] is not None: self.params['make3d'] = os.path.basename(self.params['make3d']) outfile = os.path.join(self.params['rundir'], self.params['make3d']) apEMAN.make3d(self.params['stackname'], outfile, sym=self.params['sym'], mode=self.params['mode'], hard=self.params['hard']) apEMAN.executeEmanCmd("proc3d %s %s mask=%d norm" % (outfile, outfile, self.params['mask'])) if self.params['eotest'] is True: apEMAN.make3d(self.params['oddstack'], "odd.mrc", sym=self.params['sym'], mode=self.params['mode'], hard=self.params['hard']) apEMAN.make3d(self.params['evenstack'], "even.mrc", sym=self.params['sym'], mode=self.params['mode'], hard=self.params['hard']) apEMAN.executeEmanCmd("proc3d odd.mrc even.mrc fsc=fsc.eotest") if os.path.exists(outfile): # run rmeasure apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) box = apVolume.getModelDimensions(outfile) apDisplay.printMsg('inserting density into database') symdata = apSymmetry.findSymmetry(self.params['sym']) if not symdata: apDisplay.printError('no symmetry associated with this model') modq = appiondata.Ap3dDensityData() modq['session'] = apStack.getSessionDataFromStackId( self.params['stackid']) modq['name'] = self.params['make3d'] modq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) modq['boxsize'] = box modq['mask'] = self.params['mask'] modq['pixelsize'] = apix fscres = apRecon.getResolutionFromFSCFile('fsc.eotest', box, apix, msg=True) modq['resolution'] = fscres modq['rmeasure'] = apRecon.runRMeasure(apix, outfile) modq['md5sum'] = apFile.md5sumfile(outfile) modq['maxjump'] = self.params['avgjump'] modq['sigma'] = self.params['sigma'] modq['hard'] = self.params['hard'] modq['symmetry'] = symdata modq['refineIter'] = self.params['refineiter'] if self.params['commit'] is True: modq.insert() apChimera.filterAndChimera(outfile, res=fscres, apix=apix, box=box, chimtype='snapshot', zoom=self.params['zoom'], sym=self.params['sym'], mass=self.params['mass']) else: apDisplay.printError( 'no 3d volume was generated - check the class averages:') apDisplay.printError(self.params['stackname']) apDisplay.printError( 'hard may be set too high, or avg euler jump set too low for the # of particles' ) stackstr = str(stackdata.dbid) reconstr = str(self.params['reconid']) apDisplay.printColor( "Make a new stack with only non-jumpers:\n" + "subStack.py --projectid=" + str(self.params['projectid']) + " -s " + stackstr + " \\\n " + " -k " + os.path.join(self.params['rundir'], "keep.lst") + " \\\n " + " -d 'recon " + reconstr + " sitters' -n sitters" + reconstr + " -C ", "purple")
def checkConflicts(self): ### setup correct database after we have read the project id if 'projectid' in self.params and self.params['projectid'] is not None: apDisplay.printMsg("Using split database") # use a project database newdbname = apProject.getAppionDBFromProjectId(self.params['projectid']) sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '"+newdbname+"'", "green") if self.params['stackid'] is None: apDisplay.printError("stackid was not defined") if self.params['expweight'] is False: apDisplay.printWarning("Exposure weighting is turned off, make sure this is what you want") if self.params['localavg'] is False: apDisplay.printWarning("Trajectory local averaging is turned off, make sure this is what you want") # DD processes self.dd = apDDprocess.DDStackProcessing() print self.dd # get stack data self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid']) self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True) self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) # query image qimage = self.stackparts[0]['particle']['image'] # pixel size info self.params['apix'] = apStack.getMicrographPixelSizeFromStackId(self.params['stackid']) self.params['box'] = self.stackdata['boxsize'] self.params['particleradius'] = self.params['particleradius'] / self.params['apix'] if self.params['particleradius'] > self.params['box'] / 2.0: apDisplay.printWarning("specified particle radius greater than box radius, \ setting particle radius to 0.8 * boxsize") # micrograph & frame info frames = qimage['use frames'] nframes = len(frames) if self.params['framelastali'] is None: self.params['framelastali'] = frames[-1] if self.params['framelastave'] is None: self.params['framelastave'] = frames[-1] # microscope kV self.params['kv'] = qimage['scope']['high tension']/1000.0 # query exposure per frame, if not set here if self.params['total_dose'] is not None: dose = self.params['total_dose'] else: try: dose = apDatabase.getDoseFromImageData(qimage) except: apDisplay.printError("dose not specified and not in database, please specify explicitly") if self.params['expperframe'] is None and self.params['expweight'] is True: if dose is not None: self.params['expperframe'] = dose / nframes else: apDisplay.printError("exposure per frame needs to be specified, cannot find in database") # dimensions self.params['framex'] = int(apDatabase.getDimensionsFromImageData(qimage)['x']) self.params['framey'] = int(apDatabase.getDimensionsFromImageData(qimage)['y']) # DD info self.dd.setImageData(qimage) self.dd.setDDStackRun(self.params['ddstackid']) self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def checkConflicts(self): ### make sure the necessary parameters are set if self.params['description'] is None: apDisplay.printError("enter a template description") if self.params['templatestack'] is None and self.params['clusterId'] is None and self.params['alignId'] is None: apDisplay.printError("enter a template stack file or a clusterId or an alignId") if self.params['templatetype'] is None and self.params['templatetype'] is not "clsavg" and self.params['templatetype'] is not "forward_proj": apDisplay.printError("enter the template type (i.e. class averages / forward projections)") if self.params['runname'] is None: templatestacksq = appiondata.ApTemplateStackData() templatestacks = templatestacksq.query() num_templatestacks = len(templatestacks) new_num = num_templatestacks + 1 self.params['runname'] = "templatestack"+str(new_num)+"_"+str(self.params['sessionname']) ### get apix value if (self.params['apix'] is None and self.params['clusterId'] is None and self.params['alignId'] is None): apDisplay.printError("Enter value for angstroms per pixel") elif (self.params['apix'] is None and self.params['clusterId'] is not None or self.params['alignId'] is not None): if self.params['clusterId'] is not None: clusterdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterId']) self.params['apix'] = clusterdata['clusterrun']['pixelsize'] else: aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignId']) self.params['apix'] = aligndata['pixelsize'] ### get boxsize if not specified if self.params['boxsize'] is None and self.params['templatestack'] is not None: emancmd = "iminfo "+self.params['templatestack'] proc = subprocess.Popen(emancmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) results = proc.stdout proc.wait() for line in results: res = re.search("([0-9]+)x([0-9]+)x([0-9])", line) if res: num1 = int(res.groups()[0]) num2 = int(res.groups()[1]) if num1 == num2: self.params['boxsize'] = num1 elif self.params['boxsize'] is None and self.params['clusterId'] is not None or self.params['alignId'] is not None: if self.params['clusterId'] is not None: clusterdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterId']) self.params['boxsize'] = clusterdata['clusterrun']['boxsize'] else: aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignId']) self.params['boxsize'] = aligndata['boxsize'] ### check for session if self.params['sessionname'] is None: if self.params['clusterId'] is not None: clusterdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterId']) stackid = clusterdata['clusterrun']['alignstack']['stack'].dbid sessiondata = apStack.getSessionDataFromStackId(stackid) self.params['sessionname'] = sessiondata['name'] elif self.params['alignId'] is not None: aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignId']) stackid = aligndata['stack'].dbid sessiondata = apStack.getSessionDataFromStackId(stackid) self.params['sessionname'] = sessiondata['name'] else: apDisplay.printError("Could not find session") if self.params['templatestack'] is not None: self.params['templatestack'] = os.path.abspath(self.params['templatestack']) ### exclusions and inclusions, if modifying previous stack if self.params['clusterId'] is not None or self.params['alignId'] is not None: if self.params['first'] is None and self.params['last'] is None: if self.params['keepfile'] is None and self.params['exclude'] is None and self.params['include'] is None: apDisplay.printError("Please define either keepfile, exclude or include") elif self.params['keepfile']: self.params['keepfile'] = os.path.abspath(self.params['keepfile']) if not os.path.isfile(self.params['keepfile']): apDisplay.printError("Could not find keep file: "+self.params['keepfile']) if self.params['keepfile'] and (self.params['exclude'] is not None or self.params['include'] is not None): apDisplay.printError("Please define only either keepfile, exclude or include") if self.params['exclude'] and (self.params['keepfile'] is not None or self.params['include'] is not None): apDisplay.printError("Please define only either keepfile, exclude or include") if self.params['include'] and (self.params['exclude'] is not None or self.params['keepfile'] is not None): apDisplay.printError("Please define only either keepfile, exclude or include")
def getProjectIdFromStackId(stackid): sessiondata = apStack.getSessionDataFromStackId(stackid) projectid = getProjectIdFromSessionData(sessiondata) return projectid
def start(self): self.params['outputstack'] = os.path.join(self.params['rundir'], self.params['stackname']) particles,self.params['refineiter'] = getParticleInfo(self.params['reconid'], self.params['iter']) stackdata = particles[0]['particle']['stack'] stack = os.path.join(stackdata['path']['path'], stackdata['name']) classes,cstats = determineClasses(particles) rejectlst=[] if self.params['sigma'] is not None: cutoff=cstats['meanquality']+self.params['sigma']*cstats['stdquality'] apDisplay.printMsg("Cutoff = "+str(cutoff)) rejectlst = self.removePtclsByQualityFactor(particles, rejectlst, cutoff) if self.params['avgjump'] is not None: rejectlst = self.removePtclsByJumps(particles, rejectlst) if self.params['rejectlst']: rejectlst = removePtclsByLst(rejectlst, self.params) classkeys=classes.keys() classkeys.sort() classnum=0 totalptcls=0 keepfile=open('keep.lst','w') keepfile.write('#LST\n') reject=open('reject.lst','w') reject.write('#LST\n') apDisplay.printMsg("Processing classes") #loop through classes for key in classkeys: # file to hold particles of this class clsfile=open('clstmp.lst','w') clsfile.write('#LST\n') classnum+=1 if classnum%10 == 1: apDisplay.printMsg(str(classnum)+" of "+(str(len(classkeys)))) images=EMAN.EMData() #loop through particles in class nptcls=0 for ptcl in classes[key]['particles']: if ptcl['mirror']: mirror=1 else: mirror=0 rot=ptcl['euler3'] rot=rot*math.pi/180 if ptcl['particle']['particleNumber'] not in rejectlst: l = '%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1, stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror) keepfile.write(l) clsfile.write(l) totalptcls+=1 nptcls+=1 else: reject.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1, stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror)) #if ptcl['quality_factor']>cstats['meanquality']+3*cstats['stdquality']: # high.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1, # stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror)) clsfile.close() if nptcls<1: continue if self.params['skipavg'] is False: makeClassAverages('clstmp.lst', self.params['outputstack'], classes[key], self.params) if self.params['eotest'] is True: self.makeEvenOddClasses('clstmp.lst',classes[key]) apDisplay.printMsg("\n") reject.close() keepfile.close() os.remove('clstmp.lst') # make 3d density file if specified: if self.params['make3d'] is not None: self.params['make3d']=os.path.basename(self.params['make3d']) outfile = os.path.join(self.params['rundir'],self.params['make3d']) apEMAN.make3d(self.params['stackname'], outfile, sym=self.params['sym'], mode=self.params['mode'], hard=self.params['hard']) apEMAN.executeEmanCmd("proc3d %s %s mask=%d norm" % (outfile,outfile,self.params['mask'])) if self.params['eotest'] is True: apEMAN.make3d(self.params['oddstack'],"odd.mrc", sym=self.params['sym'],mode=self.params['mode'],hard=self.params['hard']) apEMAN.make3d(self.params['evenstack'],"even.mrc", sym=self.params['sym'],mode=self.params['mode'],hard=self.params['hard']) apEMAN.executeEmanCmd("proc3d odd.mrc even.mrc fsc=fsc.eotest") if os.path.exists(outfile): # run rmeasure apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) box = apVolume.getModelDimensions(outfile) apDisplay.printMsg('inserting density into database') symdata=apSymmetry.findSymmetry(self.params['sym']) if not symdata: apDisplay.printError('no symmetry associated with this model') modq=appiondata.Ap3dDensityData() modq['session']=apStack.getSessionDataFromStackId(self.params['stackid']) modq['name']=self.params['make3d'] modq['path']=appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) modq['boxsize']=box modq['mask']=self.params['mask'] modq['pixelsize']=apix fscres=apRecon.getResolutionFromFSCFile('fsc.eotest',box,apix,msg=True) modq['resolution']=fscres modq['rmeasure']=apRecon.runRMeasure(apix,outfile) modq['md5sum']=apFile.md5sumfile(outfile) modq['maxjump']=self.params['avgjump'] modq['sigma']=self.params['sigma'] modq['hard']=self.params['hard'] modq['symmetry']=symdata modq['refineIter']=self.params['refineiter'] if self.params['commit'] is True: modq.insert() apChimera.filterAndChimera(outfile, res=fscres, apix=apix, box=box, chimtype='snapshot', zoom=self.params['zoom'], sym=self.params['sym'], mass=self.params['mass']) else: apDisplay.printError('no 3d volume was generated - check the class averages:') apDisplay.printError(self.params['stackname']) apDisplay.printError('hard may be set too high, or avg euler jump set too low for the # of particles') stackstr = str(stackdata.dbid) reconstr = str(self.params['reconid']) apDisplay.printColor("Make a new stack with only non-jumpers:\n" +"subStack.py --projectid="+str(self.params['projectid'])+" -s "+stackstr+" \\\n " +" -k "+os.path.join(self.params['rundir'],"keep.lst")+" \\\n " +" -d 'recon "+reconstr+" sitters' -n sitters"+reconstr+" -C ", "purple")