def medianVolume(self):
volpath = os.path.join(self.params['rundir'], "volumes/*a.mrc")
mrcfiles = glob.glob(volpath)
volumes = []
for filename in mrcfiles:
if os.path.isfile(filename):
vol = mrc.read(filename)
print filename, vol.shape
volumes.append(vol)
volarray = numpy.asarray(volumes, dtype=numpy.float32)
try:
medarray = numpy.median(volarray, axis=0)
except:
medarray = numpy.median(volarray)
medfile = os.path.join(self.params['rundir'], "volumes/medianVolume.mrc")
print medfile, medarray.shape
mrc.write(medarray, medfile)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
uploadcmd = ( ("uploadModel.py --projectid=%d --session=%s --file=%s "
+"--apix=%.3f --sym=%s --name=satmedian-recon%d.mrc --res=30 --description='%s %d'")
%(self.params['projectid'], sessiondata['name'], medfile,
apix, self.params['symmname'], self.params['reconid'],
"SAT selected median volume for recon", self.params['reconid'], ) )
apDisplay.printColor(uploadcmd, "purple")
f = open("upload.sh", "w")
f.write(uploadcmd+"\n")
f.close()
def medianVolume(self):
volpath = os.path.join(self.params['rundir'], "volumes/*a.mrc")
mrcfiles = glob.glob(volpath)
volumes = []
for filename in mrcfiles:
if os.path.isfile(filename):
vol = mrc.read(filename)
print filename, vol.shape
volumes.append(vol)
volarray = numpy.asarray(volumes, dtype=numpy.float32)
try:
medarray = numpy.median(volarray, axis=0)
except:
medarray = numpy.median(volarray)
medfile = os.path.join(self.params['rundir'], "volumes/medianVolume.mrc")
print medfile, medarray.shape
mrc.write(medarray, medfile)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
uploadcmd = ( ("uploadModel.py --projectid=%d --session=%s --file=%s "
+"--apix=%.3f --sym=%s --name=satmedian-recon%d.mrc --res=30 --description='%s %d'")
%(self.params['projectid'], sessiondata['name'], medfile,
apix, self.params['symmname'], self.params['reconid'],
"SAT selected median volume for recon", self.params['reconid'], ) )
apDisplay.printColor(uploadcmd, "purple")
f = open("upload.sh", "w")
f.write(uploadcmd+"\n")
f.close()
def useClusterForTemplate(self):
apDisplay.printMsg("Using clustering stack to make templates")
clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
self.params['apix'] = clusterstackdata['clusterrun']['alignstack']['pixelsize']
stackfile = os.path.join(clusterstackdata['path']['path'], clusterstackdata['avg_imagicfile'])
self.params['templatelist'] = []
stackid = clusterstackdata['clusterrun']['alignstack']['stack'].dbid
sessiondata = apStack.getSessionDataFromStackId(stackid)
self.params['sessionname'] = sessiondata['name']
### check to see if stackimagenum is within the boundary of the stack
numpart = apFile.numImagesInStack(stackfile)
for i in self.params['imgnums'].split(","):
partnum = int(i)
if partnum > numpart or partnum < 0:
apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart))
apDisplay.printMsg("Extracting image %d from cluster stack" % partnum)
templatename = "cluster%d-average%d.mrc"%(self.params['clusterid'],partnum)
abstemplatepath= os.path.join(clusterstackdata['path']['path'], templatename)
### run proc2d with params['stackimgnum']
emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum)
### create template
apDisplay.printMsg("creating "+templatename)
apEMAN.executeEmanCmd(emancmd)
if os.path.isfile(abstemplatepath):
self.params['templatelist'].append(abstemplatepath)
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(
self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '" + newdbname + "'",
"green")
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(
self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(
self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(
self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def useClusterForTemplate(self):
apDisplay.printMsg("Using clustering stack to make templates")
clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
self.params['apix'] = clusterstackdata['clusterrun']['alignstack']['pixelsize']
stackfile = os.path.join(clusterstackdata['path']['path'], clusterstackdata['avg_imagicfile'])
self.params['templatelist'] = []
stackid = clusterstackdata['clusterrun']['alignstack']['stack'].dbid
sessiondata = apStack.getSessionDataFromStackId(stackid)
self.params['sessionname'] = sessiondata['name']
### check to see if stackimagenum is within the boundary of the stack
numpart = apFile.numImagesInStack(stackfile)
for i in self.params['imgnums'].split(","):
partnum = int(i)
if partnum > numpart or partnum < 0:
apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart))
apDisplay.printMsg("Extracting image %d from cluster stack" % partnum)
templatename = "cluster%d-average%d.mrc"%(self.params['clusterid'],partnum)
abstemplatepath= os.path.join(clusterstackdata['path']['path'], templatename)
### run proc2d with params['stackimgnum']
emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum)
### create template
apDisplay.printMsg("creating "+templatename)
apEMAN.executeEmanCmd(emancmd)
if os.path.isfile(abstemplatepath):
self.params['templatelist'].append(abstemplatepath)
def getFormattedStackInfo(stackdata, format):
sessiondata = apStack.getSessionDataFromStackId(stackdata.dbid)
stackroot = os.path.splitext(stackdata["name"])[0]
instackdir = stackdata["path"]["path"]
outstackdir = os.path.join(
appionbasepath, sessiondata["name"], "stacks", os.path.basename(stackdata["path"]["path"]) + "_" + format
)
return stackroot, instackdir, outstackdir
def getFormattedStackInfo(stackdata, format):
sessiondata = apStack.getSessionDataFromStackId(stackdata.dbid)
stackroot = os.path.splitext(stackdata['name'])[0]
instackdir = stackdata['path']['path']
outstackdir = os.path.join(
appionbasepath, sessiondata['name'], 'stacks',
os.path.basename(stackdata['path']['path']) + '_' + format)
return stackroot, instackdir, outstackdir
def insertOtrRun(self, volfile):
### setup resolutions
fscresq = appiondata.ApResolutionData()
fscresq["type"] = "fsc"
fscresq["half"] = self.fscresolution
fscresq["fscfile"] = "fscdata" + self.timestamp + ".fsc"
rmeasureq = appiondata.ApResolutionData()
rmeasureq["type"] = "rmeasure"
rmeasureq["half"] = self.rmeasureresolution
rmeasureq["fscfile"] = None
### insert rct run data
otrrunq = appiondata.ApOtrRunData()
otrrunq["runname"] = self.params["runname"]
tempstr = ""
for cnum in self.classlist:
tempstr += str(cnum) + ","
classliststr = tempstr[:-1]
otrrunq["classnums"] = classliststr
otrrunq["numiter"] = self.params["numiters"]
otrrunq["euleriter"] = self.params["refineiters"]
otrrunq["maskrad"] = self.params["radius"]
otrrunq["lowpassvol"] = self.params["lowpassvol"]
otrrunq["highpasspart"] = self.params["highpasspart"]
otrrunq["median"] = self.params["median"]
otrrunq["description"] = self.params["description"]
otrrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
otrrunq["alignstack"] = self.alignstackdata
otrrunq["tiltstack"] = apStack.getOnlyStackData(self.params["tiltstackid"])
otrrunq["numpart"] = self.numpart
otrrunq["fsc_resolution"] = fscresq
otrrunq["rmeasure_resolution"] = rmeasureq
if self.params["commit"] is True:
otrrunq.insert()
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq["otrrun"] = otrrunq
densq["path"] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile)))
densq["name"] = os.path.basename(volfile)
densq["hidden"] = False
densq["norm"] = True
densq["symmetry"] = appiondata.ApSymmetryData.direct_query(25)
densq["pixelsize"] = apStack.getStackPixelSizeFromStackId(self.params["partstackid"]) * self.params["bin"]
densq["boxsize"] = self.getBoxSize()
densq["lowpass"] = self.params["lowpassvol"]
densq["highpass"] = self.params["highpasspart"]
densq["mask"] = self.params["radius"]
# densq['iterid'] = self.params['numiters']
densq["description"] = self.params["description"]
densq["resolution"] = self.fscresolution
densq["rmeasure"] = self.rmeasureresolution
densq["session"] = apStack.getSessionDataFromStackId(self.params["tiltstackid"])
densq["md5sum"] = apFile.md5sumfile(volfile)
if self.params["commit"] is True:
densq.insert()
return
def insertOtrRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq['type'] = "fsc" fscresq['half'] = self.fscresolution fscresq['fscfile'] = "fscdata"+self.timestamp+".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq['type'] = "rmeasure" rmeasureq['half'] = self.rmeasureresolution rmeasureq['fscfile'] = None ### insert rct run data otrrunq = appiondata.ApOtrRunData() otrrunq['runname'] = self.params['runname'] tempstr = "" for cnum in self.classlist: tempstr += str(cnum)+"," classliststr = tempstr[:-1] otrrunq['classnums'] = classliststr otrrunq['numiter'] = self.params['numiters'] otrrunq['euleriter'] = self.params['refineiters'] otrrunq['maskrad'] = self.params['radius'] otrrunq['lowpassvol'] = self.params['lowpassvol'] otrrunq['highpasspart'] = self.params['highpasspart'] otrrunq['median'] = self.params['median'] otrrunq['description'] = self.params['description'] otrrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) otrrunq['alignstack'] = self.alignstackdata otrrunq['tiltstack'] = apStack.getOnlyStackData(self.params['tiltstackid']) otrrunq['numpart'] = self.numpart otrrunq['fsc_resolution'] = fscresq otrrunq['rmeasure_resolution'] = rmeasureq if self.params['commit'] is True: otrrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['otrrun'] = otrrunq densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25) densq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] densq['boxsize'] = self.getBoxSize() densq['lowpass'] = self.params['lowpassvol'] densq['highpass'] = self.params['highpasspart'] densq['mask'] = self.params['radius'] #densq['iterid'] = self.params['numiters'] densq['description'] = self.params['description'] densq['resolution'] = self.fscresolution densq['rmeasure'] = self.rmeasureresolution densq['session'] = apStack.getSessionDataFromStackId(self.params['tiltstackid']) densq['md5sum'] = apFile.md5sumfile(volfile) if self.params['commit'] is True: densq.insert() return
def insertRefinementRunData(self, iteration, reference_number=1):
''' fills in database entry object for ApRefineRunData table '''
### fill in ApMultiModelRefineRunData object, if valid
if self.multiModelRefinementRun is True:
multimodelq = appiondata.ApMultiModelRefineRunData()
multimodelq['runname'] = self.params['runname']
multimodelq[
'REF|projectdata|projects|project'] = apProject.getProjectIdFromStackId(
self.runparams['stackid'])
multimodelq['session'] = apStack.getSessionDataFromStackId(
self.runparams['stackid'])
multimodelq['num_refinements'] = self.runparams[
'NumberOfReferences']
self.multimodelq = multimodelq
### fill in ApRefineRunData object
runq = appiondata.ApRefineRunData()
runq['runname'] = self.params['runname']
runq['stack'] = self.stackdata
runq['reference_number'] = reference_number
earlyresult = runq.query(results=1) # check unique run
if earlyresult:
apDisplay.printWarning(
"Run already exists in the database.\nIdentical data will not be reinserted"
)
paramdescription = self.params['description']
if not paramdescription:
paramdescription = None
runq['job'] = self.params['jobinfo']
if len(self.modeldata) > 1:
runq['initialModel'] = self.modeldata[reference_number - 1]
else:
runq['initialModel'] = self.modeldata[0]
runq['package'] = self.package
runq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.resultspath))
runq['description'] = paramdescription
runq['num_iter'] = self.runparams['numiter']
if self.multiModelRefinementRun is True:
runq['multiModelRefineRun'] = self.multimodelq
result = runq.query(results=1)
# save run entry in the parameters
if result:
self.refinerunq = result[0]
elif self.params['commit'] is True:
apDisplay.printMsg(
"Refinement Run was not found, setting to inserted values")
self.refinerunq = runq
else:
apDisplay.printWarning(
"Refinement Run was not found, setting to 'None'")
self.refinerunq = None
return
def checkConflicts(self):
if self.params["reconid"] is None:
apDisplay.printError("enter a reconstruction ID from the database")
if self.params["mask"] is None:
apDisplay.printError("enter a mask radius")
if self.params["iter"] is None:
apDisplay.printError("enter an iteration for the final Eulers")
self.params["stackid"] = apStack.getStackIdFromRecon(self.params["reconid"])
self.params["sessionname"] = apStack.getSessionDataFromStackId(self.params["stackid"])
def checkConflicts(self):
### make sure the necessary parameters are set
if self.params['description'] is None:
apDisplay.printError("enter a template description")
if self.params['diam'] is None:
apDisplay.printError("enter the particle diameter in Angstroms")
### make sure we have something
if (self.params['template'] is None
and self.params['stackid'] is None
and self.params['clusterid'] is None
and self.params['alignid'] is None
):
apDisplay.printError("enter a template root name (wild cards are valid)")
### check if apix is needed
if (self.params['apix'] is None
and self.params['stackid'] is None
and self.params['clusterid'] is None
and self.params['alignid'] is None
):
apDisplay.printError("enter a pixel size")
### check for session
if self.params['sessionname'] is None:
if self.params['alignid'] is not None:
alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignid'])
stackid = alignstackdata['stack'].dbid
sessiondata = apStack.getSessionDataFromStackId(stackid)
self.params['sessionname'] = sessiondata['name']
elif self.params['stackid'] is not None:
stackid = self.params['stackid']
sessiondata = apStack.getSessionDataFromStackId(stackid)
self.params['sessionname'] = sessiondata['name']
elif self.params['alignid'] is not None:
clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
stackid = clusterstackdata['clusterrun']['alignstack']['stack'].dbid
sessiondata = apStack.getSessionDataFromStackId(stackid)
self.params['sessionname'] = sessiondata['name']
if self.params['sessionname'] is None:
apDisplay.printError("Could not find session")
if self.params['template'] is not None:
self.params['template'] = os.path.abspath(self.params['template'])
def checkConflicts(self):
### make sure the necessary parameters are set
if self.params['description'] is None:
apDisplay.printError("enter a template description")
if self.params['diam'] is None:
apDisplay.printError("enter the particle diameter in Angstroms")
### make sure we have something
if (self.params['template'] is None
and self.params['stackid'] is None
and self.params['clusterid'] is None
and self.params['alignid'] is None
):
apDisplay.printError("enter a template root name (wild cards are valid)")
### check if apix is needed
if (self.params['apix'] is None
and self.params['stackid'] is None
and self.params['clusterid'] is None
and self.params['alignid'] is None
):
apDisplay.printError("enter a pixel size")
### check for session
if self.params['sessionname'] is None:
if self.params['alignid'] is not None:
alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignid'])
stackid = alignstackdata['stack'].dbid
sessiondata = apStack.getSessionDataFromStackId(stackid)
self.params['sessionname'] = sessiondata['name']
elif self.params['stackid'] is not None:
stackid = self.params['stackid']
sessiondata = apStack.getSessionDataFromStackId(stackid)
self.params['sessionname'] = sessiondata['name']
elif self.params['alignid'] is not None:
clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
stackid = clusterstackdata['clusterrun']['alignstack']['stack'].dbid
sessiondata = apStack.getSessionDataFromStackId(stackid)
self.params['sessionname'] = sessiondata['name']
if self.params['sessionname'] is None:
apDisplay.printError("Could not find session")
if self.params['template'] is not None:
self.params['template'] = os.path.abspath(self.params['template'])
def checkConflicts(self):
if self.params['reconid'] is None:
apDisplay.printError("enter a reconstruction ID from the database")
if self.params['mask'] is None:
apDisplay.printError("enter a mask radius")
if self.params['iter'] is None:
apDisplay.printError("enter an iteration for the final Eulers")
self.params['stackid'] = apStack.getStackIdFromRecon(
self.params['reconid'])
self.params['sessionname'] = apStack.getSessionDataFromStackId(
self.params['stackid'])['name']
def start(self):
### check for existing run
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
selectrundata = selectrunq.query(readimages=False)
if selectrundata:
apDisplay.printError("Runname already exists")
### stack data
stackdata = apStack.getOnlyStackData(self.params['stackid'])
### stack particles
stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
stackparts.reverse()
### selection run for first particle
oldselectrun = stackparts[0]['particle']['selectionrun']
### set selection run
manualparamsq = appiondata.ApManualParamsData()
manualparamsq['diam'] = self.getDiamFromSelectionRun(oldselectrun)
manualparamsq['oldselectionrun'] = oldselectrun
manualparamsq['trace'] = False
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['hidden'] = False
selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
selectrunq['session'] = apStack.getSessionDataFromStackId(self.params['stackid'])
selectrunq['manparams'] = manualparamsq
### insert particles
apDisplay.printMsg("Inserting particles into database")
count = 0
t0 = time.time()
startmem = mem.active()
numpart = len(stackparts)
for stackpart in stackparts:
count += 1
if count > 10 and count%100 == 0:
perpart = (time.time()-t0)/float(count+1)
apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"%
(count, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart),
apDisplay.timeString(perpart*(numpart-count))), "blue")
oldpartdata = stackpart['particle']
newpartq = appiondata.ApParticleData(initializer=oldpartdata)
newpartq['selectionrun'] = selectrunq
if self.params['commit'] is True:
newpartq.insert()
apDisplay.printMsg("Completed in %s"%(apDisplay.timeString(time.time()-t0)))
def getSessionData(self):
sessiondata = None
if 'sessionname' in self.params and self.params['sessionname'] is not None:
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
if not sessiondata and 'stackid' in self.params:
from appionlib import apStack
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
if not sessiondata:
### works with only canonical session names
s = re.search('/([0-9][0-9][a-z][a-z][a-z][0-9][0-9][^/]*)/', self.params['rundir'])
if s:
self.params['sessionname'] = s.groups()[0]
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
return sessiondata
def insertRefinementRunData(self, iteration, reference_number=1):
""" fills in database entry object for ApRefineRunData table """
### fill in ApMultiModelRefineRunData object, if valid
if self.multiModelRefinementRun is True:
multimodelq = appiondata.ApMultiModelRefineRunData()
multimodelq["runname"] = self.params["runname"]
multimodelq["REF|projectdata|projects|project"] = apProject.getProjectIdFromStackId(
self.runparams["stackid"]
)
multimodelq["session"] = apStack.getSessionDataFromStackId(self.runparams["stackid"])
multimodelq["num_refinements"] = self.runparams["NumberOfReferences"]
self.multimodelq = multimodelq
### fill in ApRefineRunData object
runq = appiondata.ApRefineRunData()
runq["runname"] = self.params["runname"]
runq["stack"] = self.stackdata
runq["reference_number"] = reference_number
earlyresult = runq.query(results=1) # check unique run
if earlyresult:
apDisplay.printWarning("Run already exists in the database.\nIdentical data will not be reinserted")
paramdescription = self.params["description"]
if not paramdescription:
paramdescription = None
runq["job"] = self.params["jobinfo"]
if len(self.modeldata) > 1:
runq["initialModel"] = self.modeldata[reference_number - 1]
else:
runq["initialModel"] = self.modeldata[0]
runq["package"] = self.package
runq["path"] = appiondata.ApPathData(path=os.path.abspath(self.resultspath))
runq["description"] = paramdescription
runq["num_iter"] = self.runparams["numiter"]
if self.multiModelRefinementRun is True:
runq["multiModelRefineRun"] = self.multimodelq
result = runq.query(results=1)
# save run entry in the parameters
if result:
self.refinerunq = result[0]
elif self.params["commit"] is True:
apDisplay.printMsg("Refinement Run was not found, setting to inserted values")
self.refinerunq = runq
else:
apDisplay.printWarning("Refinement Run was not found, setting to 'None'")
self.refinerunq = None
return
def useStackForTemplate(self):
apDisplay.printMsg("Using stack to make templates")
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
self.params['sessionname'] = sessiondata['name']
self.params['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
stackdata = apStack.getOnlyStackData(self.params['stackid'])
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
self.params['templatelist'] = []
if self.params['avgstack'] is True:
### averaging all particles together
apDisplay.printMsg("Averaging all images in stack")
templatename = "stack%d-average.mrc" % (self.params['stackid'])
oldtemplatepath = os.path.join(stackdata['path']['path'],
"average.mrc")
abstemplatepath = os.path.join(stackdata['path']['path'],
templatename)
if os.path.isfile(oldtemplatepath):
shutil.copy(oldtemplatepath, abstemplatepath)
else:
### average all images using proc2d
emancmd = "proc2d %s %s average" % (stackfile, abstemplatepath)
apEMAN.executeEmanCmd(emancmd)
if os.path.isfile(abstemplatepath):
self.params['templatelist'] = [abstemplatepath]
elif self.params['imgnums'] is not None:
### check to see if stackimagenum is within the boundary of the stack
numpart = apFile.numImagesInStack(stackfile)
for i in self.params['imgnums'].split(","):
partnum = int(i)
if partnum > numpart or partnum < 0:
apDisplay.printError(
"'imgnums' is NOT within the boundary of the stack: %d > %d"
% (partnum, numpart))
apDisplay.printMsg("Extracting image %d from stack" % partnum)
templatename = "stack%d-particle%d.mrc" % (
self.params['stackid'], partnum)
abstemplatepath = os.path.join(self.params['rundir'],
templatename)
### run proc2d with params['stackimgnum']
emancmd = "proc2d %s %s first=%d last=%d" % (
stackfile, abstemplatepath, partnum, partnum)
### create template
apDisplay.printMsg("creating " + templatename)
apEMAN.executeEmanCmd(emancmd)
if os.path.isfile(abstemplatepath):
self.params['templatelist'].append(abstemplatepath)
def getSessionData(self):
sessiondata = None
if "sessionname" in self.params and self.params["sessionname"] is not None:
sessiondata = apDatabase.getSessionDataFromSessionName(self.params["sessionname"])
if not sessiondata and "stackid" in self.params:
from appionlib import apStack
sessiondata = apStack.getSessionDataFromStackId(self.params["stackid"])
if not sessiondata:
### works with only canonical session names
s = re.search("/([0-9][0-9][a-z][a-z][a-z][0-9][0-9][^/]*)/", self.params["rundir"])
if s:
self.params["sessionname"] = s.groups()[0]
sessiondata = apDatabase.getSessionDataFromSessionName(self.params["sessionname"])
return sessiondata
def useStackForTemplate(self):
apDisplay.printMsg("Using stack to make templates")
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
self.params['sessionname'] = sessiondata['name']
self.params['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
stackdata = apStack.getOnlyStackData(self.params['stackid'])
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
self.params['templatelist'] = []
if self.params['avgstack'] is True:
### averaging all particles together
apDisplay.printMsg("Averaging all images in stack")
templatename = "stack%d-average.mrc"%(self.params['stackid'])
oldtemplatepath = os.path.join(stackdata['path']['path'], "average.mrc")
abstemplatepath = os.path.join(stackdata['path']['path'], templatename)
if os.path.isfile(oldtemplatepath):
shutil.copy(oldtemplatepath, abstemplatepath)
else:
### average all images using proc2d
emancmd = "proc2d %s %s average" % (stackfile, abstemplatepath)
apEMAN.executeEmanCmd(emancmd)
if os.path.isfile(abstemplatepath):
self.params['templatelist'] = [abstemplatepath]
elif self.params['imgnums'] is not None:
### check to see if stackimagenum is within the boundary of the stack
numpart = apFile.numImagesInStack(stackfile)
for i in self.params['imgnums'].split(","):
partnum = int(i)
if partnum > numpart or partnum < 0:
apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart))
apDisplay.printMsg("Extracting image %d from stack" % partnum)
templatename = "stack%d-particle%d.mrc"%(self.params['stackid'],partnum)
abstemplatepath= os.path.join(self.params['rundir'], templatename)
### run proc2d with params['stackimgnum']
emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum)
### create template
apDisplay.printMsg("creating "+templatename)
apEMAN.executeEmanCmd(emancmd)
if os.path.isfile(abstemplatepath):
self.params['templatelist'].append(abstemplatepath)
def uploadDensity(self, volfile):
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile)))
densq['name'] = os.path.basename(volfile)
densq['hidden'] = False
densq['norm'] = True
densq['symmetry'] = self.symmdata
densq['pixelsize'] = self.clusterstackdata['clusterrun']['pixelsize']
densq['boxsize'] = self.clusterstackdata['clusterrun']['boxsize']
#densq['lowpass'] = None
#densq['highpass'] = None
densq['mask'] = self.params['mask']
densq['description'] = self.params['description']+"- From eman start-"+self.params['method']
#densq['resolution'] = None
densq['session'] = apStack.getSessionDataFromStackId(self.params['stackid'])
densq['md5sum'] = apFile.md5sumfile(volfile)
densq['eman'] = self.params['method']
if self.params['commit'] is True:
densq.insert()
return
def uploadDensity(self, volfile): ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = self.symmdata densq['pixelsize'] = self.clusterstackdata['clusterrun']['pixelsize'] densq['boxsize'] = self.clusterstackdata['clusterrun']['boxsize'] #densq['lowpass'] = None #densq['highpass'] = None densq['mask'] = self.params['mask'] densq['description'] = self.params['description']+"- From eman start-"+self.params['method'] #densq['resolution'] = None densq['session'] = apStack.getSessionDataFromStackId(self.params['stackid']) densq['md5sum'] = apFile.md5sumfile(volfile) densq['eman'] = self.params['method'] if self.params['commit'] is True: densq.insert() return
def start(self):
### check for existing run
selectrunq = appiondata.ApSelectionRunData()
selectrunq["name"] = self.params["runname"]
selectrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
selectrundata = selectrunq.query(readimages=False)
if selectrundata:
apDisplay.printError("Runname already exists")
if self.params["ddstack"]:
self.other_ddstack_used = []
self.dd = apDDprocess.DDStackProcessing()
self.dd.setDDStackRun(self.params["ddstack"])
self.newddstackrun = self.dd.getDDStackRun(show_msg=True)
### stack data
stackdata = apStack.getOnlyStackData(self.params["stackid"])
### stack particles
stackparts = apStack.getStackParticlesFromId(self.params["stackid"], msg=True)
stackparts.reverse()
### selection run for first particle
oldselectrun = stackparts[0]["particle"]["selectionrun"]
### set selection run
manualparamsq = appiondata.ApManualParamsData()
manualparamsq["diam"] = self.getDiamFromSelectionRun(oldselectrun)
manualparamsq["oldselectionrun"] = oldselectrun
manualparamsq["trace"] = False
selectrunq = appiondata.ApSelectionRunData()
selectrunq["name"] = self.params["runname"]
selectrunq["hidden"] = False
selectrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
selectrunq["session"] = apStack.getSessionDataFromStackId(self.params["stackid"])
selectrunq["manparams"] = manualparamsq
### insert particles
apDisplay.printMsg("Inserting particles into database")
count = 0
t0 = time.time()
startmem = mem.active()
numpart = len(stackparts)
for stackpart in stackparts:
count += 1
if count > 10 and count % 100 == 0:
perpart = (time.time() - t0) / float(count + 1)
apDisplay.printColor(
"part %d of %d :: %.1fM mem :: %s/part :: %s remain"
% (
count,
numpart,
(mem.active() - startmem) / 1024.0,
apDisplay.timeString(perpart),
apDisplay.timeString(perpart * (numpart - count)),
),
"blue",
)
oldpartdata = stackpart["particle"]
newpartq = appiondata.ApParticleData(initializer=oldpartdata)
newpartq["selectionrun"] = selectrunq
if self.params["ddstack"]:
newimagedata = self.getNewImageFromDDStack(oldpartdata["image"])
if newimagedata is False:
# no pick transferred
continue
newpartq["image"] = newimagedata
if self.params["commit"] is True:
newpartq.insert()
apDisplay.printMsg("Completed in %s" % (apDisplay.timeString(time.time() - t0)))
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(
self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '" + newdbname + "'",
"green")
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['expweight'] is False:
apDisplay.printWarning(
"Exposure weighting is turned off, make sure this is what you want"
)
if self.params['localavg'] is False:
apDisplay.printWarning(
"Trajectory local averaging is turned off, make sure this is what you want"
)
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(
self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(
self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(
self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# pixel size info
self.params['apix'] = apStack.getMicrographPixelSizeFromStackId(
self.params['stackid'])
self.params['box'] = self.stackdata['boxsize']
self.params['particleradius'] = self.params[
'particleradius'] / self.params['apix']
if self.params['particleradius'] > self.params['box'] / 2.0:
apDisplay.printWarning(
"specified particle radius greater than box radius, \
setting particle radius to 0.8 * boxsize")
# micrograph & frame info
frames = qimage['use frames']
nframes = len(frames)
if self.params['framelastali'] is None:
self.params['framelastali'] = frames[-1]
if self.params['framelastave'] is None:
self.params['framelastave'] = frames[-1]
# microscope kV
self.params['kv'] = qimage['scope']['high tension'] / 1000.0
# query exposure per frame, if not set here
if self.params['total_dose'] is not None:
dose = self.params['total_dose']
else:
try:
dose = apDatabase.getDoseFromImageData(qimage)
except:
apDisplay.printError(
"dose not specified and not in database, please specify explicitly"
)
if self.params['expperframe'] is None and self.params[
'expweight'] is True:
if dose is not None:
self.params['expperframe'] = dose / nframes
else:
apDisplay.printError(
"exposure per frame needs to be specified, cannot find in database"
)
# dimensions
self.params['framex'] = int(
apDatabase.getDimensionsFromImageData(qimage)['x'])
self.params['framey'] = int(
apDatabase.getDimensionsFromImageData(qimage)['y'])
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def getFormattedStackInfo(stackdata,format):
sessiondata = apStack.getSessionDataFromStackId(stackdata.dbid)
stackroot = os.path.splitext(stackdata['name'])[0]
instackdir = stackdata['path']['path']
outstackdir = os.path.join(appionbasepath,sessiondata['name'],'stacks',os.path.basename(stackdata['path']['path'])+'_'+format)
return stackroot,instackdir,outstackdir
def start(self):
self.params['outputstack'] = os.path.join(self.params['rundir'],
self.params['stackname'])
particles, self.params['refineiter'] = getParticleInfo(
self.params['reconid'], self.params['iter'])
stackdata = particles[0]['particle']['stack']
stack = os.path.join(stackdata['path']['path'], stackdata['name'])
classes, cstats = determineClasses(particles)
rejectlst = []
if self.params['sigma'] is not None:
cutoff = cstats[
'meanquality'] + self.params['sigma'] * cstats['stdquality']
apDisplay.printMsg("Cutoff = " + str(cutoff))
rejectlst = self.removePtclsByQualityFactor(
particles, rejectlst, cutoff)
if self.params['avgjump'] is not None:
rejectlst = self.removePtclsByJumps(particles, rejectlst)
if self.params['rejectlst']:
rejectlst = removePtclsByLst(rejectlst, self.params)
classkeys = classes.keys()
classkeys.sort()
classnum = 0
totalptcls = 0
keepfile = open('keep.lst', 'w')
keepfile.write('#LST\n')
reject = open('reject.lst', 'w')
reject.write('#LST\n')
apDisplay.printMsg("Processing classes")
#loop through classes
for key in classkeys:
# file to hold particles of this class
clsfile = open('clstmp.lst', 'w')
clsfile.write('#LST\n')
classnum += 1
if classnum % 10 == 1:
apDisplay.printMsg(
str(classnum) + " of " + (str(len(classkeys))))
images = EMAN.EMData()
#loop through particles in class
nptcls = 0
for ptcl in classes[key]['particles']:
if ptcl['mirror']:
mirror = 1
else:
mirror = 0
rot = ptcl['euler3']
rot = rot * math.pi / 180
if ptcl['particle']['particleNumber'] not in rejectlst:
l = '%d\t%s\t%f,\t%f,%f,%f,%d\n' % (
ptcl['particle']['particleNumber'] - 1, stack,
ptcl['quality_factor'], rot, ptcl['shiftx'],
ptcl['shifty'], mirror)
keepfile.write(l)
clsfile.write(l)
totalptcls += 1
nptcls += 1
else:
reject.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' %
(ptcl['particle']['particleNumber'] - 1,
stack, ptcl['quality_factor'], rot,
ptcl['shiftx'], ptcl['shifty'], mirror))
#if ptcl['quality_factor']>cstats['meanquality']+3*cstats['stdquality']:
# high.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1,
# stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror))
clsfile.close()
if nptcls < 1:
continue
if self.params['skipavg'] is False:
makeClassAverages('clstmp.lst', self.params['outputstack'],
classes[key], self.params)
if self.params['eotest'] is True:
self.makeEvenOddClasses('clstmp.lst', classes[key])
apDisplay.printMsg("\n")
reject.close()
keepfile.close()
os.remove('clstmp.lst')
# make 3d density file if specified:
if self.params['make3d'] is not None:
self.params['make3d'] = os.path.basename(self.params['make3d'])
outfile = os.path.join(self.params['rundir'],
self.params['make3d'])
apEMAN.make3d(self.params['stackname'],
outfile,
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d %s %s mask=%d norm" %
(outfile, outfile, self.params['mask']))
if self.params['eotest'] is True:
apEMAN.make3d(self.params['oddstack'],
"odd.mrc",
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.make3d(self.params['evenstack'],
"even.mrc",
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d odd.mrc even.mrc fsc=fsc.eotest")
if os.path.exists(outfile):
# run rmeasure
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
box = apVolume.getModelDimensions(outfile)
apDisplay.printMsg('inserting density into database')
symdata = apSymmetry.findSymmetry(self.params['sym'])
if not symdata:
apDisplay.printError('no symmetry associated with this model')
modq = appiondata.Ap3dDensityData()
modq['session'] = apStack.getSessionDataFromStackId(
self.params['stackid'])
modq['name'] = self.params['make3d']
modq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
modq['boxsize'] = box
modq['mask'] = self.params['mask']
modq['pixelsize'] = apix
fscres = apRecon.getResolutionFromFSCFile('fsc.eotest',
box,
apix,
msg=True)
modq['resolution'] = fscres
modq['rmeasure'] = apRecon.runRMeasure(apix, outfile)
modq['md5sum'] = apFile.md5sumfile(outfile)
modq['maxjump'] = self.params['avgjump']
modq['sigma'] = self.params['sigma']
modq['hard'] = self.params['hard']
modq['symmetry'] = symdata
modq['refineIter'] = self.params['refineiter']
if self.params['commit'] is True:
modq.insert()
apChimera.filterAndChimera(outfile,
res=fscres,
apix=apix,
box=box,
chimtype='snapshot',
zoom=self.params['zoom'],
sym=self.params['sym'],
mass=self.params['mass'])
else:
apDisplay.printError(
'no 3d volume was generated - check the class averages:')
apDisplay.printError(self.params['stackname'])
apDisplay.printError(
'hard may be set too high, or avg euler jump set too low for the # of particles'
)
stackstr = str(stackdata.dbid)
reconstr = str(self.params['reconid'])
apDisplay.printColor(
"Make a new stack with only non-jumpers:\n" +
"subStack.py --projectid=" + str(self.params['projectid']) +
" -s " + stackstr + " \\\n " + " -k " +
os.path.join(self.params['rundir'], "keep.lst") + " \\\n " +
" -d 'recon " + reconstr + " sitters' -n sitters" + reconstr +
" -C ", "purple")
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['expweight'] is False:
apDisplay.printWarning("Exposure weighting is turned off, make sure this is what you want")
if self.params['localavg'] is False:
apDisplay.printWarning("Trajectory local averaging is turned off, make sure this is what you want")
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# pixel size info
self.params['apix'] = apStack.getMicrographPixelSizeFromStackId(self.params['stackid'])
self.params['box'] = self.stackdata['boxsize']
self.params['particleradius'] = self.params['particleradius'] / self.params['apix']
if self.params['particleradius'] > self.params['box'] / 2.0:
apDisplay.printWarning("specified particle radius greater than box radius, \
setting particle radius to 0.8 * boxsize")
# micrograph & frame info
frames = qimage['use frames']
nframes = len(frames)
if self.params['framelastali'] is None:
self.params['framelastali'] = frames[-1]
if self.params['framelastave'] is None:
self.params['framelastave'] = frames[-1]
# microscope kV
self.params['kv'] = qimage['scope']['high tension']/1000.0
# query exposure per frame, if not set here
if self.params['total_dose'] is not None:
dose = self.params['total_dose']
else:
try:
dose = apDatabase.getDoseFromImageData(qimage)
except:
apDisplay.printError("dose not specified and not in database, please specify explicitly")
if self.params['expperframe'] is None and self.params['expweight'] is True:
if dose is not None:
self.params['expperframe'] = dose / nframes
else:
apDisplay.printError("exposure per frame needs to be specified, cannot find in database")
# dimensions
self.params['framex'] = int(apDatabase.getDimensionsFromImageData(qimage)['x'])
self.params['framey'] = int(apDatabase.getDimensionsFromImageData(qimage)['y'])
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def checkConflicts(self):
### make sure the necessary parameters are set
if self.params['description'] is None:
apDisplay.printError("enter a template description")
if self.params['templatestack'] is None and self.params['clusterId'] is None and self.params['alignId'] is None:
apDisplay.printError("enter a template stack file or a clusterId or an alignId")
if self.params['templatetype'] is None and self.params['templatetype'] is not "clsavg" and self.params['templatetype'] is not "forward_proj":
apDisplay.printError("enter the template type (i.e. class averages / forward projections)")
if self.params['runname'] is None:
templatestacksq = appiondata.ApTemplateStackData()
templatestacks = templatestacksq.query()
num_templatestacks = len(templatestacks)
new_num = num_templatestacks + 1
self.params['runname'] = "templatestack"+str(new_num)+"_"+str(self.params['sessionname'])
### get apix value
if (self.params['apix'] is None and self.params['clusterId'] is None and self.params['alignId'] is None):
apDisplay.printError("Enter value for angstroms per pixel")
elif (self.params['apix'] is None and self.params['clusterId'] is not None or self.params['alignId'] is not None):
if self.params['clusterId'] is not None:
clusterdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterId'])
self.params['apix'] = clusterdata['clusterrun']['pixelsize']
else:
aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignId'])
self.params['apix'] = aligndata['pixelsize']
### get boxsize if not specified
if self.params['boxsize'] is None and self.params['templatestack'] is not None:
emancmd = "iminfo "+self.params['templatestack']
proc = subprocess.Popen(emancmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
results = proc.stdout
proc.wait()
for line in results:
res = re.search("([0-9]+)x([0-9]+)x([0-9])", line)
if res:
num1 = int(res.groups()[0])
num2 = int(res.groups()[1])
if num1 == num2:
self.params['boxsize'] = num1
elif self.params['boxsize'] is None and self.params['clusterId'] is not None or self.params['alignId'] is not None:
if self.params['clusterId'] is not None:
clusterdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterId'])
self.params['boxsize'] = clusterdata['clusterrun']['boxsize']
else:
aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignId'])
self.params['boxsize'] = aligndata['boxsize']
### check for session
if self.params['sessionname'] is None:
if self.params['clusterId'] is not None:
clusterdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterId'])
stackid = clusterdata['clusterrun']['alignstack']['stack'].dbid
sessiondata = apStack.getSessionDataFromStackId(stackid)
self.params['sessionname'] = sessiondata['name']
elif self.params['alignId'] is not None:
aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignId'])
stackid = aligndata['stack'].dbid
sessiondata = apStack.getSessionDataFromStackId(stackid)
self.params['sessionname'] = sessiondata['name']
else:
apDisplay.printError("Could not find session")
if self.params['templatestack'] is not None:
self.params['templatestack'] = os.path.abspath(self.params['templatestack'])
### exclusions and inclusions, if modifying previous stack
if self.params['clusterId'] is not None or self.params['alignId'] is not None:
if self.params['first'] is None and self.params['last'] is None:
if self.params['keepfile'] is None and self.params['exclude'] is None and self.params['include'] is None:
apDisplay.printError("Please define either keepfile, exclude or include")
elif self.params['keepfile']:
self.params['keepfile'] = os.path.abspath(self.params['keepfile'])
if not os.path.isfile(self.params['keepfile']):
apDisplay.printError("Could not find keep file: "+self.params['keepfile'])
if self.params['keepfile'] and (self.params['exclude'] is not None or self.params['include'] is not None):
apDisplay.printError("Please define only either keepfile, exclude or include")
if self.params['exclude'] and (self.params['keepfile'] is not None or self.params['include'] is not None):
apDisplay.printError("Please define only either keepfile, exclude or include")
if self.params['include'] and (self.params['exclude'] is not None or self.params['keepfile'] is not None):
apDisplay.printError("Please define only either keepfile, exclude or include")
def getProjectIdFromStackId(stackid):
sessiondata = apStack.getSessionDataFromStackId(stackid)
projectid = getProjectIdFromSessionData(sessiondata)
return projectid
def start(self):
self.params['outputstack'] = os.path.join(self.params['rundir'], self.params['stackname'])
particles,self.params['refineiter'] = getParticleInfo(self.params['reconid'], self.params['iter'])
stackdata = particles[0]['particle']['stack']
stack = os.path.join(stackdata['path']['path'], stackdata['name'])
classes,cstats = determineClasses(particles)
rejectlst=[]
if self.params['sigma'] is not None:
cutoff=cstats['meanquality']+self.params['sigma']*cstats['stdquality']
apDisplay.printMsg("Cutoff = "+str(cutoff))
rejectlst = self.removePtclsByQualityFactor(particles, rejectlst, cutoff)
if self.params['avgjump'] is not None:
rejectlst = self.removePtclsByJumps(particles, rejectlst)
if self.params['rejectlst']:
rejectlst = removePtclsByLst(rejectlst, self.params)
classkeys=classes.keys()
classkeys.sort()
classnum=0
totalptcls=0
keepfile=open('keep.lst','w')
keepfile.write('#LST\n')
reject=open('reject.lst','w')
reject.write('#LST\n')
apDisplay.printMsg("Processing classes")
#loop through classes
for key in classkeys:
# file to hold particles of this class
clsfile=open('clstmp.lst','w')
clsfile.write('#LST\n')
classnum+=1
if classnum%10 == 1:
apDisplay.printMsg(str(classnum)+" of "+(str(len(classkeys))))
images=EMAN.EMData()
#loop through particles in class
nptcls=0
for ptcl in classes[key]['particles']:
if ptcl['mirror']:
mirror=1
else:
mirror=0
rot=ptcl['euler3']
rot=rot*math.pi/180
if ptcl['particle']['particleNumber'] not in rejectlst:
l = '%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1,
stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror)
keepfile.write(l)
clsfile.write(l)
totalptcls+=1
nptcls+=1
else:
reject.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1,
stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror))
#if ptcl['quality_factor']>cstats['meanquality']+3*cstats['stdquality']:
# high.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1,
# stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror))
clsfile.close()
if nptcls<1:
continue
if self.params['skipavg'] is False:
makeClassAverages('clstmp.lst', self.params['outputstack'], classes[key], self.params)
if self.params['eotest'] is True:
self.makeEvenOddClasses('clstmp.lst',classes[key])
apDisplay.printMsg("\n")
reject.close()
keepfile.close()
os.remove('clstmp.lst')
# make 3d density file if specified:
if self.params['make3d'] is not None:
self.params['make3d']=os.path.basename(self.params['make3d'])
outfile = os.path.join(self.params['rundir'],self.params['make3d'])
apEMAN.make3d(self.params['stackname'], outfile, sym=self.params['sym'], mode=self.params['mode'], hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d %s %s mask=%d norm" % (outfile,outfile,self.params['mask']))
if self.params['eotest'] is True:
apEMAN.make3d(self.params['oddstack'],"odd.mrc", sym=self.params['sym'],mode=self.params['mode'],hard=self.params['hard'])
apEMAN.make3d(self.params['evenstack'],"even.mrc", sym=self.params['sym'],mode=self.params['mode'],hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d odd.mrc even.mrc fsc=fsc.eotest")
if os.path.exists(outfile):
# run rmeasure
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
box = apVolume.getModelDimensions(outfile)
apDisplay.printMsg('inserting density into database')
symdata=apSymmetry.findSymmetry(self.params['sym'])
if not symdata:
apDisplay.printError('no symmetry associated with this model')
modq=appiondata.Ap3dDensityData()
modq['session']=apStack.getSessionDataFromStackId(self.params['stackid'])
modq['name']=self.params['make3d']
modq['path']=appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
modq['boxsize']=box
modq['mask']=self.params['mask']
modq['pixelsize']=apix
fscres=apRecon.getResolutionFromFSCFile('fsc.eotest',box,apix,msg=True)
modq['resolution']=fscres
modq['rmeasure']=apRecon.runRMeasure(apix,outfile)
modq['md5sum']=apFile.md5sumfile(outfile)
modq['maxjump']=self.params['avgjump']
modq['sigma']=self.params['sigma']
modq['hard']=self.params['hard']
modq['symmetry']=symdata
modq['refineIter']=self.params['refineiter']
if self.params['commit'] is True:
modq.insert()
apChimera.filterAndChimera(outfile, res=fscres, apix=apix, box=box,
chimtype='snapshot', zoom=self.params['zoom'], sym=self.params['sym'], mass=self.params['mass'])
else:
apDisplay.printError('no 3d volume was generated - check the class averages:')
apDisplay.printError(self.params['stackname'])
apDisplay.printError('hard may be set too high, or avg euler jump set too low for the # of particles')
stackstr = str(stackdata.dbid)
reconstr = str(self.params['reconid'])
apDisplay.printColor("Make a new stack with only non-jumpers:\n"
+"subStack.py --projectid="+str(self.params['projectid'])+" -s "+stackstr+" \\\n "
+" -k "+os.path.join(self.params['rundir'],"keep.lst")+" \\\n "
+" -d 'recon "+reconstr+" sitters' -n sitters"+reconstr+" -C ", "purple")
