def printTimeStats(self, name, timelist):
if len(timelist) < 2:
return
meantime = self.stats['timesum']/float(self.stats['count'])
timearray = numpy.array(timelist, dtype=numpy.float64)
apDisplay.printColor("%s: %s (%.2f percent)"%
(name, apDisplay.timeString(timearray.mean(), timearray.std()), 100*timearray.mean()/meantime), "blue")
def runKerdenSOM(self, indata):
"""
From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/KerDenSOM
KerDenSOM stands for "Kernel Probability Density Estimator Self-Organizing Map".
It maps a set of high dimensional input vectors into a two-dimensional grid.
"""
apDisplay.printMsg("Running KerDen SOM")
outstamp = os.path.join(self.params['rundir'], self.timestamp)
kerdencmd = ( "xmipp_classify_kerdensom -verb 1 -i %s -o %s -xdim %d -ydim %d -saveclusters "%
(indata, outstamp, self.params['xdim'], self.params['ydim'])
)
### convergence criteria
if self.params['converge'] == "fast":
kerdencmd += " -eps 1e-5 "
elif self.params['converge'] == "slow":
kerdencmd += " -eps 1e-9 "
else:
kerdencmd += " -eps 1e-7 "
apDisplay.printColor(kerdencmd, "cyan")
proc = subprocess.Popen(kerdencmd, shell=True)
proc.wait()
time.sleep(1)
return
def boxParticlesFromImage(self, imgdata, partdatas, shiftdata):
### convert database particle data to coordinates and write boxfile
boxfile = os.path.join(self.params['rundir'], imgdata['filename']+".box")
parttree, boxedpartdatas = apBoxer.processParticleData(imgdata, self.boxsize,
partdatas, shiftdata, boxfile, rotate=self.params['rotate'])
if self.params['boxfiles']:
### quit and return, boxfile created, now process next image
return None, None, None
### check if we have particles again
if len(partdatas) == 0 or len(parttree) == 0:
apDisplay.printColor(self.shortname+" has no remaining particles and has been rejected\n","cyan")
return None, None, None
### set up output file path
imgstackfile = os.path.join(self.params['rundir'], self.shortname+".hed")
if (self.is_dd_stack and (self.params['nframe'] or self.params['driftlimit'])
and not self.params['phaseflipped'] and not self.params['rotate']):
# If processing on whole image is not needed, it is more efficient to use mmap to box frame stack
apDisplay.printMsg("boxing "+str(len(parttree))+" particles into temp file: "+imgstackfile)
framelist = self.dd.getFrameList(self.params)
apBoxer.boxerFrameStack(self.dd.framestackpath, parttree, imgstackfile, self.boxsize, framelist)
else:
self._boxParticlesFromImage(imgdata, parttree, imgstackfile)
partmeantree = self.calculateParticleStackStats(imgstackfile, boxedpartdatas)
imgstackfile = self.postProcessParticleStack(imgdata, imgstackfile, boxedpartdatas, len(parttree))
return boxedpartdatas, imgstackfile, partmeantree
def createNewSession(self):
apDisplay.printColor("Creating a new session", "cyan")
### get user data
userdata = self.getUserData()
sessionq = leginon.leginondata.SessionData()
sessionq['name'] = self.params['sessionname']
sessionq['image path'] = self.leginonimagedir
sessionq['frame path'] = self.leginonframedir
sessionq['comment'] = self.params['description']
sessionq['user'] = userdata
sessionq['hidden'] = False
projectdata = leginon.projectdata.projects.direct_query(self.params['projectid'])
projectexpq = leginon.projectdata.projectexperiments()
projectexpq['project'] = projectdata
projectexpq['session'] = sessionq
if self.params['commit'] is True:
projectexpq.insert()
self.sessiondata = sessionq
apDisplay.printColor("Created new session %s"%(self.params['sessionname']), "cyan")
return
def medianVolume(self):
volpath = os.path.join(self.params['rundir'], "volumes/*a.mrc")
mrcfiles = glob.glob(volpath)
volumes = []
for filename in mrcfiles:
if os.path.isfile(filename):
vol = mrc.read(filename)
print filename, vol.shape
volumes.append(vol)
volarray = numpy.asarray(volumes, dtype=numpy.float32)
try:
medarray = numpy.median(volarray, axis=0)
except:
medarray = numpy.median(volarray)
medfile = os.path.join(self.params['rundir'], "volumes/medianVolume.mrc")
print medfile, medarray.shape
mrc.write(medarray, medfile)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
uploadcmd = ( ("uploadModel.py --projectid=%d --session=%s --file=%s "
+"--apix=%.3f --sym=%s --name=satmedian-recon%d.mrc --res=30 --description='%s %d'")
%(self.params['projectid'], sessiondata['name'], medfile,
apix, self.params['symmname'], self.params['reconid'],
"SAT selected median volume for recon", self.params['reconid'], ) )
apDisplay.printColor(uploadcmd, "purple")
f = open("upload.sh", "w")
f.write(uploadcmd+"\n")
f.close()
def createSpiderFile(self):
"""
takes the stack file and creates a spider file ready for processing
"""
emancmd = "proc2d "
if not os.path.isfile(self.stack['file']):
apDisplay.printError("stackfile does not exist: "+self.stack['file'])
emancmd += self.stack['file']+" "
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "apix="+str(self.stack['apix'])+" "
if self.params['lowpass'] > 0:
emancmd += "lp="+str(self.params['lowpass'])+" "
if self.params['highpass'] > 0:
emancmd += "hp="+str(self.params['highpass'])+" "
if self.params['bin'] > 1:
clipboxsize = self.boxsize*self.params['bin']
emancmd += "shrink="+str(self.params['bin'])+" "
emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" "
emancmd += "last="+str(self.params['numpart']-1)+" "
emancmd += "spider edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
return spiderstack
def postLoopFunctions(self):
### Delete CTF corrected images
if self.params['keepall'] is False:
pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*.dwn.mrc')
apFile.removeFilePattern(pattern)
### remove Ace2 images
pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*mrc.corrected.mrc')
apFile.removeFilePattern(pattern)
### remove Spider images
if self.params['fliptype'] == 'spiderimage':
pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*_out.spi')
apFile.removeFilePattern(pattern)
pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*_tf.spi')
apFile.removeFilePattern(pattern)
if self.noimages is True:
return
stackpath = os.path.join(self.params['rundir'], self.params['single'])
### delete this after testing
apStack.averageStack(stack = stackpath)
### Create Stack Mean Plot
if self.params['commit'] is True and self.params['meanplot'] is True:
stackid = apStack.getStackIdFromPath(stackpath)
if stackid is not None:
apStackMeanPlot.makeStackMeanPlot(stackid)
apDisplay.printColor("Timing stats", "blue")
self.printTimeStats("Batch Boxer", self.batchboxertimes)
self.printTimeStats("Ctf Correction", self.ctftimes)
self.printTimeStats("Stack Merging", self.mergestacktimes)
self.printTimeStats("Mean/Std Read", self.meanreadtimes)
self.printTimeStats("DB Insertion", self.insertdbtimes)
def getDiffResForOverfocus(radii=None, cs=2e-3, volts=120000):
"""
given Cs and kV, determine the initial resolution where the difference between
overfocus and underfocus is clearly visible.
value returned in Angstroms, but radii must be in meters
"""
if debug is True:
print "getDiffResForOverfocus()"
if debug is True:
apDisplay.printColor("getDiffRes radii: 1/%.2fA --> 1/%.2fA"%(1/radii[1]*1e10, 1/radii[-1]*1e10), "cyan")
t0 = time.time()
checkParams(focus1=1.0e-6, focus2=1.0e-6, cs=cs, volts=volts, ampconst=0.0, failParams=False)
lamb = ctftools.getTEMLambda(volts)
s = radii
pi = math.pi
csgamma = 2*pi*0.25*cs*(lamb**3)*(s**4)
#over/under-focus difference is visible when Cs component is greater than 0.05
index = numpy.searchsorted(csgamma, 0.03)
diffres = 1.0/radii[index-1]*1e10
apDisplay.printColor("Overfocus/Underfocus difference resolution is: 1/%.2fA"%(diffres), "cyan")
if debug is True:
print "difference resolution complete in %.9f sec"%(time.time()-t0)
return diffres
def insertAlignParticlesIntoDatabase(self, level):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
D=self.getClassificationAtLevel(level)
for ref in D:
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = ref+1
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
if ref in self.resdict:
refq['ssnr_resolution'] = self.resdict[ref]
### setup particle info ... NOTE: ALIGNMENT PARAMETERS ARE NOT SAVED IN XMIPP 2.4
for partnum in D[ref]:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(partnum)+1
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(self.runparams['stackid'], int(partnum)+1) ### particle numbering starts with 0!!!!!!!
alignpartq['stackpart'] = stackpartdata
alignpartq['ref'] = refq
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in "
+apDisplay.timeString(time.time()-t0), "cyan")
return
def commonLines(stackfile, maskdiam=None, minfreq=0, maxfreq=0.25,
ang=5.0, numiter=200, outdocfile=None, numpart=None, dataext=".spi"):
"""
performs common lines on a input spider stack
"""
if numpart is None or numpart < 3:
apDisplay.printError("undefined number of particles")
if maskdiam is None:
apDisplay.printError("undefined mask diameter")
starttime = time.time()
if dataext in stackfile:
stackfile = stackfile[:-4]
randdocfile = generateRandomAngles(numpart)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider("OP",
stackfile+"@*****", #stack file
"1-%d"%(numpart), #number of particles
str(maskdiam), #mark diameter
"%.3f,%.3f"%(minfreq,maxfreq), #frequency range for line projections
str(ang), #angular increment or accuracy
str(numiter), #number of iterations
randdocfile, #random angles doc file
outdocfile, #output angles doc file
)
mySpider.close()
apDisplay.printColor("finished common lines in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def initializeRefinementUploadVariables(self):
""" untars results, reads required parameters, establishes necessary objects for upload script """
apDisplay.printColor("uploading refinement results for %s reconstruction routine" % (self.package), "cyan")
### establish directories and make an appion results directory
self.basepath = os.path.abspath(self.params["rundir"])
self.resultspath = os.path.abspath(os.path.join(self.params["rundir"], str(self.package) + "_results"))
if not os.path.isdir(self.resultspath):
os.mkdir(self.resultspath)
self.reconpath = os.path.abspath(
os.path.join(self.params["rundir"], self.runparams["reconstruction_working_dir"])
)
### get all stack parameters, map particles in reconstruction to particles in stack, get all model data
self.stackdata = apStack.getOnlyStackData(self.runparams["stackid"])
self.stackmapping = apRecon.partnum2defid(self.runparams["stackid"])
self.modeldata = []
if len(self.runparams["modelid"].split(",")) > 1:
models = self.runparams["modelid"].split(",")
for i in range(len(models)):
self.modeldata.append(appiondata.ApInitialModelData.direct_query(int(models[i])))
else:
self.modeldata.append(appiondata.ApInitialModelData.direct_query(self.runparams["modelid"]))
return
def rctParticleShift(volfile, origstackfile, eulerdocfile, iternum, numpart, pixrad, timestamp, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
starttime = time.time()
### create corresponding projections
projstackfile = "projstack%s-%03d.spi"%(timestamp, iternum)
projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext)
### clean up files
ccdocfile = "ccdocfile%s-%03d.spi"%(timestamp, iternum)
apFile.removeFile(ccdocfile)
alignstackfile = "alignstack%s-%03d.spi"%(timestamp, iternum)
apFile.removeFile(alignstackfile)
### align particles to projection
apDisplay.printMsg("Shifting particles")
crossCorrelateAndShift(origstackfile, projstackfile, alignstackfile, ccdocfile, numpart)
if not os.path.isfile(alignstackfile):
apDisplay.printError("aligned stack file not found: "+alignstackfile)
apDisplay.printColor("finished correlations in "+apDisplay.timeString(time.time()-starttime), "cyan")
return alignstackfile
def getResolutionData(self, avgpath, iternum):
evenvol = os.path.join(avgpath, "avglist3_%dp1.mrc" % (self.params["rescut"]))
oddvol = os.path.join(avgpath, "avglist3_%dp2.mrc" % (self.params["rescut"]))
evenhed = os.path.join(avgpath, "avglist3_%dp1.hed" % (self.params["rescut"]))
oddhed = os.path.join(avgpath, "avglist3_%dp2.hed" % (self.params["rescut"]))
emancmd1 = "proc3d %s %s" % (evenvol, evenhed)
apEMAN.executeEmanCmd(emancmd1, verbose=True)
emancmd2 = "proc3d %s %s" % (oddvol, oddhed)
apEMAN.executeEmanCmd(emancmd2, verbose=True)
fscfile = "fsc.eotest.%d" % (iternum)
fscpath = os.path.join(avgpath, fscfile)
emancmd3 = "proc3d %s %s fsc=%s" % (evenhed, oddhed, fscpath)
apEMAN.executeEmanCmd(emancmd3, verbose=True)
if not os.path.isfile(fscpath):
apDisplay.printWarning("Could not find FSC file: " + fscpath)
return None
f = open(fscpath, "r")
xy = f.readlines()
lines = len(xy)
boxsize = lines * 2.0
f.close()
# calculate the resolution:
halfres = apRecon.calcRes(fscpath, boxsize, self.params["step"])
apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms" % (halfres), "cyan")
# save to database
resq = appiondata.ApResolutionData()
resq["half"] = halfres
resq["fscfile"] = fscpath
return resq
def backprojectCG(stackfile, eulerdocfile, volfile, numpart, pixrad, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
### setup
starttime = time.time()
stackfile = spyder.fileFilter(stackfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(stackfile+dataext):
apDisplay.printError("stack file not found: "+stackfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(volfile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("BP CG",
stackfile+"@*****", #stack file
"1-%d"%(numpart), #number of particles
str(pixrad), #particle radius
eulerdocfile, #angle doc file
"N", #has symmetry?, does not work
volfile, #filename for volume
"%.1e,%.1f" % (1.0e-5, 0.0), #error, chi^2 limits
"%d,%d" % (25,1), #iterations, 1st derivative mode
"2000", #lambda - higher=less sensitive to noise
)
mySpider.close()
apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def start(self):
print 'wait=',self.params['wait']
max_loop_num_trials = 60 * 3
wait_time = 20
self.last_num_stacks = 0
if self.params['wait']:
num_trials = 0
while True:
limit_reached = self.loopCheckAndProcess()
if limit_reached:
apDisplay.printMsg('image limit reached. Stoping...')
break
if self.num_stacks <= self.last_num_stacks:
if num_trials >= max_loop_num_trials:
apDisplay.printColor('Checked for stack file %d times. Finishing....' % max_loop_num_trials,'magenta')
apDisplay.printMsg('Rerun this script if you know more are coming')
break
else:
num_trials += 1
else:
# reset trial number if new stack is found
num_trials = 0
apDisplay.printColor('Finished stack file checking in rundir. Will check again in %d seconds' % wait_time,'magenta')
time.sleep(wait_time)
self.last_num_stacks = self.num_stacks
else:
self.loopCheckAndProcess()
def backproject3F(stackfile, eulerdocfile, volfile, numpart, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
### setup
starttime = time.time()
stackfile = spyder.fileFilter(stackfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(stackfile+dataext):
apDisplay.printError("stack file not found: "+stackfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(volfile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("BP 3F",
stackfile+"@*****", #stack file
"1-%d"%(numpart), #number of particles
eulerdocfile, #angle doc file
"*", #input symmetry file, '*' for skip
volfile, #filename for volume
)
mySpider.close()
apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def start(self):
"""
this is the main component of the script
where all the processing is done
"""
### initialize some variables
self.runq = None
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
apDisplay.printMsg("Pixel size: %.5f"%(self.apix))
self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
apDisplay.printMsg("Box size: %d"%(self.boxsize))
self.checkResults()
self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
self.numiter = self.getNumberOfIterations()
for i in range(self.numiter):
iternum = i+1
apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green")
self.uploadIteration(iternum)
reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir'])
if reconrunid:
apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid'])
apRecon.setGoodBadParticlesFromReconId(reconrunid)
else:
apDisplay.printWarning("Could not find recon run id")
def getNumberOfIterations(self):
iternum = 0
stop = False
while stop is False:
## check if iteration is complete
iternum += 1
paramfile = "params.iter%03d.par"%(iternum)
if not os.path.isfile(paramfile):
apDisplay.printWarning("Parameter file %s is missing"%(paramfile))
stop = True
break
imagicvolume = "threed.%03da.hed"%(iternum)
if not os.path.isfile(imagicvolume):
apDisplay.printWarning("Volume file %s is missing"%(imagicvolume))
stop = True
break
combineshell = "iter%03d/frealign.iter%03d.combine.sh"%(iternum, iternum)
if not os.path.isfile(combineshell):
apDisplay.printWarning("Shell file %s is missing"%(combineshell))
stop = True
break
### set last working iteration
numiter = iternum-1
if numiter < 1:
apDisplay.printError("No iterations were found")
apDisplay.printColor("Found %d complete iterations"%(numiter), "green")
return numiter
def findLastCompletedIteration(self):
#recondir = os.path.join(self.params['rundir'], "recon")
recondir = self.params['rundir']
iternum = 0
stop = False
while stop is False:
## check if iteration is complete
iternum += 1
class1Volume = "recon_it%03d_half1_model.star"%(iternum)
class2Volume = "recon_it%03d_half2_model.star"%(iternum)
class1Volumepath = os.path.join(recondir,class1Volume)
class2Volumepath = os.path.join(recondir,class2Volume)
if not os.path.isfile(class1Volumepath) or not os.path.isfile(class2Volumepath):
apDisplay.printWarning("Model.star file %s or %s is missing"%(class1Volumepath, class2Volumepath))
stop = True
break
### set last working iteration
numiter = iternum-1
if numiter < 1:
apDisplay.printError("No iterations were found")
apDisplay.printColor("Found %d complete iterations"%(numiter), "green")
return numiter
def start(self):
### universal particle counter
self.partnum = 1
### final stack file
self.combinefile = os.path.join( self.params['rundir'], self.params['stackfilename'] )
if os.path.isfile(self.combinefile):
apDisplay.printError("A stack with name "+self.params['stackfilename']+" and path "
+self.params['rundir']+" already exists.")
### loop through stacks
for stackstr in self.params['stackids']:
stackid = int(stackstr)
### get stack data
stackdata = apStack.getOnlyStackData(stackid)
### append particle to stack file
self.appendToStack(stackdata)
if self.params['commit'] is True:
### insert stack data
apDisplay.printColor("inserting new stack particles from stackid="+str(stackid), "cyan")
self.commitStack(stackid)
else:
apDisplay.printWarning("not committing data to database")
apStack.averageStack(stack=self.combinefile)
def convertStackToSpider(self, emanstackfile, classnum):
"""
takes the stack file and creates a spider file ready for processing
"""
if not os.path.isfile(emanstackfile):
apDisplay.printError("stackfile does not exist: "+emanstackfile)
### first high pass filter particles
apDisplay.printMsg("pre-filtering particles")
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
emancmd = ("proc2d "+emanstackfile+" "+emanstackfile
+" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
+" inplace")
apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc2d "
emancmd += emanstackfile+" "
spiderstack = os.path.join(self.params['rundir'], str(classnum), "otrstack"+self.timestamp+".spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
return spiderstack
def runEoTest(self, corrSelectOdd, corrSelectEven, cnum, apshstack, apsheuler, iternum):
apshOddVolfile = os.path.join(self.params['rundir'], str(cnum), "apshVolume_Odd-%03d.spi"%(iternum))
apshEvenVolfile = os.path.join(self.params['rundir'], str(cnum), "apshVolume_Even-%03d.spi"%(iternum))
self.APSHbackProject(apshstack, apsheuler, apshOddVolfile, cnum, corrSelectOdd)
self.APSHbackProject(apshstack, apsheuler, apshEvenVolfile, cnum, corrSelectEven)
fscout = os.path.join(self.params['rundir'], str(cnum), "FSCout-%03d.spi"%(iternum))
backproject.calcFSC(apshOddVolfile, apshEvenVolfile, fscout)
### Calculate FSC - taken from Neil's RCT script
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin']
emancmd = "proc3d %s %s"%(apshEvenVolfile, apshEvenVolfile+".mrc")
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
emancmd = "proc3d %s %s"%(apshOddVolfile, apshOddVolfile+".mrc")
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
fscfile = os.path.join(self.params['rundir'], "fscdata%s.fsc"%(self.timestamp))
emancmd = "proc3d %s %s fsc=%s"%(apshEvenVolfile+".mrc", apshOddVolfile+".mrc", fscfile)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
if not os.path.isfile(fscfile):
apDisplay.printError("Even-Odd fsc calculation failed")
boxsize = self.getBoxSize()
self.fscresolution = apRecon.getResolutionFromFSCFile(fscfile, boxsize, apix, msg=True)
apDisplay.printColor( ("Final FSC resolution: %.5f" % (self.fscresolution)), "cyan")
return fscout
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def refineEllipseLoop(self, fftarray, lowerbound, upperbound):
"""
refines the following parameters, in order
ellipRatio
ellipAngle
defocus
amplitude contrast (mostly a dependent variable)
"""
self.fminCount = 0
### "self.ellipseParams" controls what self.from2Dinto1D() does
ellipRatio = self.ellipseParams['a']/self.ellipseParams['b']
ellipAlpha = self.ellipseParams['alpha']
origEllipParams = copy.deepcopy(self.ellipseParams)
self.fftarray = fftarray
self.lowerbound = lowerbound
self.upperbound = upperbound
### create function self.refineMinFunc that would return res80+res50
x0 = [ellipRatio, ellipAlpha]
maxfun = self.params['refineIter']
results = scipy.optimize.fmin(self.refineMinFunc, x0=x0, maxfun=maxfun)
ellipRatio, ellipAlpha = results
apDisplay.printColor("BEST FROM FMIN :: defRatio=%.3f < a=%.2f"
%(ellipRatio**2, math.degrees(ellipAlpha)), "blue")
self.printBestValues()
time.sleep(10)
def setDBfromProjectId(projectid, die=True):
newdbname = getAppionDBFromProjectId(projectid, die=die)
if newdbname is None:
return False
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
return True
def projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext=".spi"):
"""
project 3D volumes using given Euler angles
"""
starttime = time.time()
volfile = spyder.fileFilter(volfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
projstackfile = spyder.fileFilter(projstackfile)
if not os.path.isfile(volfile+dataext):
apDisplay.printError("volume file not found: "+volfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(projstackfile)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("PJ 3Q",
volfile, #input vol file
str(pixrad), #pixel radius
"1-%d"%(numpart), #number of particles
eulerdocfile, #Euler DOC file
projstackfile+"@*****", #output projections
)
mySpider.close()
apDisplay.printColor("finished projections in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def _startLoop(self, info):
"""
appionLoop OVERRIDE
initilizes several parameters for a new image
and checks if it is okay to start processing image
"""
if info is None:
self.stats['lastimageskipped'] = True
self.stats['skipcount'] += 1
return False
name = info['filename']
# check to see if image of the same name is already in leginon
imgq = leginon.leginondata.AcquisitionImageData(session=self.session, filename=name)
results = imgq.query(readimages=False)
if results:
apDisplay.printWarning("File %s.mrc exists at the destination" % name)
apDisplay.printWarning("Skip Uploading")
self.stats['lastimageskipped'] = True
self.stats['skipcount'] += 1
return False
#calc images left
self.stats['imagesleft'] = self.stats['imagecount'] - self.stats['count']
#only if an image was processed last
if(self.stats['lastcount'] != self.stats['count']):
apDisplay.printColor( "\nStarting image "+str(self.stats['count'])\
+" ( skip:"+str(self.stats['skipcount'])+", remain:"\
+str(self.stats['imagesleft'])+" ) file: "\
+apDisplay.short(name), "green")
self.stats['lastcount'] = self.stats['count']
self._checkMemLeak()
# check to see if image has already been processed
if self._alreadyProcessed(info):
return False
self.stats['waittime'] = 0
return True
def runChimeraScript(chimscript, xvfb=False):
if not chimscript or not os.path.isfile(chimscript):
print chimscript
apDisplay.printError("Could not find file: apChimSnapshot.py")
#apDisplay.printColor("Trying to use chimera for model imaging","cyan")
if xvfb is True:
port = apParam.resetVirtualFrameBuffer()
time.sleep(1)
if 'CHIMERA' in os.environ and os.path.isdir(os.environ['CHIMERA']):
chimpath = os.environ['CHIMERA']
os.environ['CHIMERA'] = chimpath
os.environ['CHIMERAPATH'] = os.path.join(chimpath,"share")
os.environ['LD_LIBRARY_PATH'] = os.path.join(chimpath,"lib")+":"+os.environ['LD_LIBRARY_PATH']
chimexe = os.path.join(chimpath,"bin/chimera")
if not os.path.isfile(chimexe):
apDisplay.printWarning("Could not find chimera at: "+chimexe)
else:
chimpath = None
chimexe = "chimera"
#apDisplay.printWarning("'CHIMERA' environmental variable is unset")
rendercmd = (chimexe+" --debug python:"+chimscript)
logf = open("chimeraRun.log", "a")
apDisplay.printColor("running Chimera:\n "+rendercmd, "cyan")
if xvfb is True:
print "import -verbose -display :%d -window root screencapture.png"%(port)
proc = subprocess.Popen(rendercmd, shell=True, stdout=logf, stderr=logf)
proc.wait()
logf.close()
if xvfb is True:
apParam.killVirtualFrameBuffer(port)
return
def normalizeVol(volfile, dataext=".spi"):
"""
inputs:
volume
outputs:
volume
"""
### setup
starttime = time.time()
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(volfile+dataext):
apDisplay.printError("volume file not found: "+volfile+dataext)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
### read out the statistics of the volume
mySpider.toSpider("FS x11,x12",
volfile, #filename for volume
)
mySpider.toSpider("IF(x12.LT.0.0)x12=-x12")
### set all values to positive
mySpider.toSpider("AR",
volfile, #filename for volume
"_1",
"(P1+x12)",
)
### save file
mySpider.toSpider("CP",
"_1",
volfile, #filename for volume
)
mySpider.close()
apDisplay.printColor("finished normalizing the volume to set all values to be positive"+apDisplay.timeString(time.time()-starttime), "cyan")
return
def createSpiderFile(self):
"""
takes the stack file and creates a spider file ready for processing
"""
emancmd = "proc2d "
if not os.path.isfile(self.stack["file"]):
apDisplay.printError("stackfile does not exist: " + self.stack["file"])
emancmd += self.stack["file"] + " "
spiderstack = os.path.join(self.params["rundir"], "start.spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack + " "
emancmd += "apix=" + str(self.stack["apix"]) + " "
if self.params["lowpass"] > 0:
emancmd += "lp=" + str(self.params["lowpass"]) + " "
emancmd += "last=" + str(self.params["numpart"] - 1) + " "
emancmd += "shrink=" + str(self.params["bin"]) + " "
clipsize = int(math.floor(self.stack["boxsize"] / self.params["bin"] / 2.0) * self.params["bin"] * 2)
emancmd += "clip=" + str(clipsize) + "," + str(clipsize) + " "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in " + apDisplay.timeString(time.time() - starttime), "cyan")
return spiderstack
def createIterationJobRefineFiles(self, oldprefix, newprefix, stackFile):
"""
Create multiple job files for frealign reconstruction
using the mpiexec command
"""
### create individual mpi scripts
particlePerProcess = float(
self.numpart) / self.params['totalprocs'] - 1
apDisplay.printColor(
"Using approx %.1f particles per process file, for a total of %d processes"
% (particlePerProcess, self.params['totalprocs']), "purple")
lastPart = 0
procjobfiles = []
for i in range(self.params['totalprocs']):
procNum = i + 1
inputVolFile = "../threed.%s.mrc" % (oldprefix)
firstPart = lastPart + 1
lastPart = firstPart + particlePerProcess
if lastPart > self.numpart - 2:
lastPart = self.numpart
inputParamFile = "../params.%s.par" % (oldprefix)
apParam.createDirectory(newprefix, warning=False)
jobFile = "%s/refine.%s.proc%03d.sh" % (newprefix, newprefix,
procNum)
procPrefix = "%s.proc%03d" % (newprefix, procNum)
f = open(jobFile, 'w')
f.write("#!/bin/sh\n\n")
fullpath = os.path.join(self.params['rundir'], newprefix)
f.write('cd %s\n' % fullpath)
f.write("/bin/rm -fv frealign.%s.out\n" % (procPrefix))
f.write("/bin/rm -fv outparams.%s.par\n" % (procPrefix))
f.write("/bin/rm -fv shift.%s.par\n" % (procPrefix))
f.write("mkdir /tmp/frealign\n")
f.write("rsync -vrtPhL %s /tmp/frealign/%s\n" %
(inputVolFile, os.path.basename(inputVolFile)))
f.write("rsync -vrtPhL %s /tmp/frealign/%s\n" %
(stackFile, os.path.basename(stackFile)))
f.close()
#partDiff = math.floor(lastPart) - math.floor(firstPart)
#print "proc %d: %.1f->%.1f (%d)"%(procNum, firstPart, lastPart, partDiff)
self.appendFrealignJobFile(jobFile,
inputParamFile,
inputVolFile,
stackFile,
math.floor(firstPart),
math.floor(lastPart),
procPrefix,
recon=False)
return
def applyEnvelopeAndCTF(self, stack):
### get defocus lists
numpart = self.params['projcount']
cut = int(numpart/80.0)+1
apDisplay.printMsg("%d particles per dot"%(cut))
if len(self.deflist1) == 0:
self.getListOfDefoci(numpart)
### break up particles
partlistdocfile = apXmipp.breakupStackIntoSingleFiles(stack, filetype="mrc")
t0 = time.time()
apDisplay.printMsg("Applying CTF and Envelop to particles")
### apply CTF using ACE2
ctfapplydocfile = self.applyCTFToDocFile(partlistdocfile)
### apply Envelop using ACE2
envelopdocfile = self.applyEnvelopToDocFile(ctfapplydocfile)
### correct CTF using ACE2
if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True:
ctfcorrectdocfile = self.correctCTFToDocFile(envelopdocfile)
else:
ctfcorrectdocfile = envelopdocfile
timeper = (time.time()-t0)/float(numpart)
apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green")
apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green")
### write corrected particle list to doc file
ctfpartlist = []
ctfpartlistfile = os.path.join(self.params['rundir'], "ctfpartlist.lst")
inf = open(ctfcorrectdocfile, 'r')
outf = open(ctfpartlistfile, "w")
for line in inf:
### get filename
filename = line.strip().split()[0]
if not os.path.isfile(filename):
apDisplay.printError("CTF and envelop apply failed")
ctfpartlist.append(filename)
outf.write(filename+"\t1\n")
inf.close()
outf.close()
### merge individual files into a common stack
ctfstack = os.path.join(self.params['rundir'], "ctfstack.hed")
apXmipp.gatherSingleFilesIntoStack(ctfpartlistfile, ctfstack, filetype="mrc")
if self.params['pad'] is True:
emancmd = "proc2d %s %s.clip.hed clip=%d,%d" % (ctfstack, ctfstack[:-4], self.params['box'], self.params['box'])
apParam.runCmd(emancmd, "EMAN")
shutil.move("%s.clip.hed" % ctfstack[:-4], "%s.hed" % ctfstack[:-4])
shutil.move("%s.clip.img" % ctfstack[:-4], "%s.img" % ctfstack[:-4])
return ctfstack, ctfpartlist
def evenLogSplit(self, start, end, power=1.7):
endlog = int(round(math.log(end) / math.log(power), 0))
startlog = int(round(math.log(start) / math.log(power), 0))
stacklist = []
for n in range(startlog, endlog, 1):
numparticles = round(math.pow(power, n), 0)
stacklist.append(int(numparticles))
apDisplay.printColor(
"Making stacks of the following sizes: " + str(stacklist), "cyan")
return (stacklist)
def start(self):
res = apRecon.getResolutionFromFSCFile(self.params['fscfile'],
self.params['boxsize'],
self.params['apix'],
self.params['criteria'])
apDisplay.printColor(("resolution: %.5f at criteria %.3f" %
(res, self.params['criteria'])), "cyan")
sys.stdout.write(
"%s resolution %.5f (%.3f)\n" %
(self.params['fscfile'], res, self.params['criteria']))
def runIMAGICclassify(self):
bfile = "msaclassify.job"
outfile = "classes"
apFile.removeStack(outfile)
numIters = int(self.params['numpart']*self.params['itermult'])
decrement = self.params['start']-self.params['end']
if self.params['iter']>0:
decrement /= float(self.params['iter'])
numClasses = self.params['start']-(decrement*self.params['currentiter'])
stackfile = self.params['alignedstack']
self.params['currentnumclasses'] = numClasses
f = open(bfile,'w')
f.write("#!/bin/csh -f\n")
f.write("setenv IMAGIC_BATCH 1\n")
f.write("%s/msa/classify.e <<EOF\n" % self.imagicroot)
f.write("IMAGES\n")
f.write("%s\n"%stackfile)
f.write("0\n")
f.write("%i\n"%self.params['activeeigen'])
f.write("YES\n")
f.write("%i\n"%numClasses)
f.write("classes_start\n")
f.write("EOF\n")
f.write("%s/msa/classum.e <<EOF\n" % self.imagicroot)
f.write("%s\n"%stackfile)
f.write("classes_start\n")
f.write("%s\n"%outfile)
f.write("YES\n")
f.write("NONE\n")
f.write("0\n")
if int(self.imagicversion) >= 120619:
f.write("NONE\n") # Mode of summing statistics
f.write("EOF\n")
## make eigenimage stack appion-compatible
f.write("proc2d eigenim.hed eigenim.hed inplace\n")
f.write("touch msaclassify_done.txt\n")
f.write("exit\n")
f.close()
## execute the batch file
aligntime0 = time.time()
apEMAN.executeEmanCmd("chmod 755 "+bfile)
apDisplay.printColor("Running IMAGIC .batch file: %s"%(os.path.abspath(bfile)), "cyan")
apIMAGIC.executeImagicBatchFile(os.path.abspath(bfile))
os.remove("msaclassify_done.txt")
if not os.path.exists(outfile+".hed"):
apDisplay.printError("ERROR IN IMAGIC SUBROUTINE")
apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-aligntime0), "cyan")
def insertParticlesIntoDatabase(self, stackid, partList):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for partdict in partList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = partdict['refnum']
if refq['refnum'] < 1:
apDisplay.printError("Classes must start with 1")
refq['iteration'] = self.lastiter
#refq['mrcfile'] = refbase+".mrc" ### need to create mrcs of each class???
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
if partdict['refnum'] in self.resdict:
refq['ssnr_resolution'] = self.resdict[partdict['refnum']]
refq['isac_generation'] = partdict['generation']
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partdict['partnum']
if alignpartq['partnum'] < 1:
apDisplay.printError("Particles must start with 1")
alignpartq['alignstack'] = self.alignstackdata
if self.params['alignstackid'] is None:
stackpartdata = apStack.getStackParticle(
stackid, partdict['origPartNum'])
else:
stackpartdata = apStack.getStackParticleFromAlignParticle(
self.params['alignstackid'], partdict['origPartNum'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['inplane']
alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
alignpartq['score'] = partdict['peak']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " + str(count) +
" particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def processImage(self, imgdata):
#image = self.getImage(imgdata, self.params['bin'])
self.image = imgdata['image']
# Check the image shape
imgshape = numpy.asarray(imgdata['image'].shape)
apDisplay.printMsg("MRC Image Shape prior to processing:")
print imgshape
imagepath = os.path.join(imgdata['session']['image path'],
imgdata['filename'] + ".mrc")
# Convert the MRC image to a jpg for find_mask.py
apDisplay.printMsg("Converting mrc image to jpg.")
jpg_dir = os.path.join(self.params['rundir'], "jpgs")
jpg_image = os.path.join(jpg_dir, imgdata['filename'] + ".jpg")
apParam.createDirectory(jpg_dir, warning=False)
pyami.numpil.write(self.image, jpg_image)
if (self.params['test']):
self.outfile = os.path.join(self.params['rundir'], "tests",
imgdata['filename'] + "_mask.jpg")
else:
self.outfile = os.path.join(self.params['rundir'], "masks",
imgdata['filename'] + "_mask.jpg")
downsample = str(self.params['downsample'])
compsizethresh = str(self.params['compsizethresh'])
adapthresh = str(self.params['adapthresh'])
dilation = str(self.params['dilation'])
erosion = str(self.params['erosion'])
blur = str(self.params['blur'])
options = " --downsample=" + downsample + " --compsizethresh=" + compsizethresh + " --adapthresh=" + adapthresh + " --blur=" + blur + " --dilation=" + dilation + " --erosion=" + erosion
commandline = ("source " + self.activatepath +
"; python `which find_mask.py` --ifile=" + jpg_image +
" --ofile=" + self.outfile + options + "\n")
# Test with test image
#commandline = ( "source /opt/em_hole_finder/env/bin/activate; python /opt/em_hole_finder/find_mask_amber.py \n" )
### run command
apDisplay.printMsg("running em hole finder " + time.asctime())
apDisplay.printColor(commandline, "purple")
if True:
proc = subprocess.Popen(commandline, shell=True)
else:
outf = open("automasker.out", "a")
errf = open("automasker.err", "a")
proc = subprocess.Popen(commandline,
shell=True,
stderr=errf,
stdout=outf)
proc.wait()
def rejectImage(self, imgdata):
shortname = apDisplay.short(imgdata['filename'])
if self.params['mag']:
if not apDatabase.checkMag(imgdata, self.params['mag']):
apDisplay.printColor(
shortname + " was not at the specific magnification",
"cyan")
return False
return True
def onClose(self):
if self.params['fulltomoId']:
apDisplay.printMsg('------------------------')
apDisplay.printWarning(
'Repeat the same script to create more subtomogram')
apDisplay.printColor(
'etomo_subrecon.py --session=%s --projectid=%d --fulltomoid=%d --description="" --commit --expId=%d --jobtype=%s --runname=etomosub'
% (self.params['sessionname'], self.params['projectid'],
self.params['fulltomoId'], self.params['expid'],
'etomo_subrecon'), 'cyan')
apDisplay.printMsg('------------------------')
def estimateIterTime(self):
secperiter = 0.12037
calctime = ((self.params['numpart'] / 1000.0) *
(self.stack['boxsize'] / self.params['bin'])**2 /
self.params['angle']**2 / float(self.nproc) * secperiter)
if self.params['mirror'] is True:
calctime *= 2.0
self.params['estimatedtime'] = calctime
apDisplay.printColor(
"Estimated first iteration time: " +
apDisplay.timeString(calctime), "purple")
def insertParticlesIntoDatabase(self, partlist, insert=False):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for particle in partlist:
count += 1
### get all particle parameters
partnum = count
rotation = particle[0] * -1 ### in spider negative rotation is clockwise
shiftx = particle[1]
shifty = particle[2]
ccc = particle[3]
refnum = int(particle[4])
mirror = particle[5]
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['imagicfile'] = "references.hed"
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['templatestack'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId'])
reffile = os.path.join(self.params['rundir'], refq['imagicfile'])
if not os.path.isfile(reffile):
apDisplay.printError("could not find reference file: "+reffile)
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(self.params['stackId'], partnum)
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = shiftx
alignpartq['yshift'] = shifty
alignpartq['rotation'] = rotation
alignpartq['mirror'] = mirror
alignpartq['ref'] = refq
alignpartq['correlation'] = ccc
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in "
+apDisplay.timeString(time.time()-t0), "cyan")
return
def generateCTF1d(radii=None,
focus=1.0e-6,
cs=2e-3,
volts=120000,
ampconst=0.07,
failParams=False,
overfocus=False):
"""
calculates a CTF function based on the input details
Use SI units: meters, radians, volts
Underfocus is postive (defocused)
"""
if debug is True:
print "generateCTF1dFromRadii()"
if radii is None:
radii = generateRadii1d(numpoints=256, pixelsize=1e-10)
if debug is True:
apDisplay.printColor(
"generateCTF radii: 1/%.2fA --> 1/%.2fA" %
(1 / radii[1] * 1e10, 1 / radii[-1] * 1e10), "cyan")
t0 = time.time()
checkParams(focus1=focus,
focus2=focus,
cs=cs,
volts=volts,
ampconst=ampconst,
failParams=failParams)
lamb = ctftools.getTEMLambda(volts)
s = radii
pi = math.pi
if overfocus is True:
focus = -1.0 * focus
gamma = -0.5 * pi * cs * (lamb**3) * (s**4) + pi * focus * lamb * (s**2)
if overfocus is True:
gamma = -1.0 * gamma
A = ampconst
B = math.sqrt(1.0 - ampconst**2)
prectf = A * numpy.cos(gamma) + B * numpy.sin(gamma)
ctf = prectf**2
if debug is True:
print "generate 1D ctf complete in %.9f sec" % (time.time() - t0)
return ctf
def setFileName(self):
if self.params['name'] is None:
### assign provided name
basename = "emdb%s-%s" % (self.params['emdbid'], self.timestamp)
else:
### clean up provided name
basename = os.path.splitext(os.path.basename(
self.params['name']))[0]
self.params['name'] = os.path.join(self.params['rundir'], basename)
apDisplay.printColor("Naming EMDB model: " + self.params['name'],
"cyan")
return
def close(self):
self.onClose()
loadavg = os.getloadavg()[0]
if loadavg > 2.0:
apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
time.sleep(loadavg**2)
apParam.closeFunctionLog(functionname=self.functionname,
logfile=self.logfile, msg=(not self.quiet))
if self.quiet is False:
apDisplay.printMsg("Ended at "+time.strftime("%a, %d %b %Y %H:%M:%S"))
apDisplay.printMsg("Memory increase during run: %.3f MB"%((mem.active()-self.startmem)/1024.0))
apDisplay.printColor("Total run time:\t"+apDisplay.timeString(time.time()-self.t0),"green")
def satAverageCmd(self):
keepfile = os.path.join(self.params['rundir'],
"keeplist-tot" + self.datastr + ".lst")
newname = "recon%d_cut%d_iter%d.hed" % (
self.params['reconid'], self.params['cutrange'] * 10, self.iternum)
cmd = ("satAverage.py " + " --projectid=" +
str(self.params['projectid']) + " --reconid=" +
str(self.params['reconid']) + " \\\n --mask=62 --iter=" +
str(self.iternum) + " \\\n --stackname=" + newname +
" \\\n --keep-list=" + keepfile + " \n")
print "New satAverage.py Command:"
apDisplay.printColor(cmd, "purple")
def oldLogSplit(self, start,end,divisions):
end=math.log(end)
start=math.log(start)
incr=(end-start)/divisions
val=start
stacklist=[]
for n in range(0, divisions):
nptcls=int(round(math.exp(val)))
stacklist.append(nptcls)
val+=incr
apDisplay.printColor("Making stacks of the following sizes: "+str(stacklist), "cyan")
return(stacklist)
def onClose(self):
if self.fullrundata:
apDisplay.printMsg('------------------------')
apDisplay.printWarning(
'To create full tomogram reconstruction and commit the result to database with these sampled tomograms, you need to use etomo_recon.py to start eTOMO and continue at "Tomogram Positioning" in %s with the .edf file by running this AppionScript:'
)
apDisplay.printColor(
'etomo_recon.py --session=%s --projectid=%d --samplerunid=%d --description="" --commit --expId=%d --jobtype=%s'
% (self.params['sessionname'], self.params['projectid'],
self.fullrundata.dbid, self.params['expid'], 'etomo_recon'),
'cyan')
apDisplay.printMsg('------------------------')
def readFileToPeakTree(self):
apDisplay.printMsg("Reading file: " + self.params['filename'])
f = open(self.params['filename'], "r")
count = 0
imgfilename2peaklist = {}
apDisplay.printMsg("Reading input file")
for line in f:
### format: x <tab> y <tab> filename
sline = line.strip()
cols = sline.split('\t')
### must have 3 columns
if len(cols) == 4:
partid, xcoord, ycoord, filename = cols
elif len(cols) == 3:
xcoord, ycoord, filename = cols
else:
continue
### check to make sure our x,y are integers, if not skip to next line in file
try:
xcoord = int(xcoord)
ycoord = int(ycoord)
except:
continue
### create new list for new files
if not filename in imgfilename2peaklist.keys():
imgfilename2peaklist[filename] = []
peakdict = {
'diameter': self.params['diam'],
'xcoord': xcoord,
'ycoord': ycoord,
'peakarea': 10,
}
count += 1
if count % 10 == 0:
sys.stderr.write(".")
imgfilename2peaklist[filename].append(peakdict)
sys.stderr.write("done\n")
f.close()
apDisplay.printColor(
"Found %d particles in %d images" %
(count, len(imgfilename2peaklist.keys())), "cyan")
if count == 0:
apDisplay.printError("No particles were found")
return imgfilename2peaklist
def onClose(self):
if self.fulltomodata:
apDisplay.printMsg('------------------------')
apDisplay.printWarning(
'To create sub tomogram reconstruction and commit the result to database with this full tomogram, you need to use etomo_subrecon.py to start eTOMO and continue at "Post-Processing" with the .edf file by running this AppionScript:'
)
apDisplay.printColor(
'etomo_subrecon.py --session=%s --projectid=%d --fulltomoid=%d --description="" --commit --expId=%d --jobtype=%s --runname=etomosub'
% (self.params['sessionname'], self.params['projectid'],
self.fulltomodata.dbid, self.params['expid'],
'etomo_subrecon'), 'cyan')
apDisplay.printMsg('------------------------')
def runrefine(self):
### setup Xmipp command
recontime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -vol " + os.path.join(self.params['rundir'], self.voldocfile) +
" -iter " + str(self.params['maxiter']) + " -o " +
os.path.join(self.params['rundir'], "part" + self.timestamp) +
" -psi_step " + str(self.params['psi']) + " -ang " +
str(self.params['phi']))
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-18 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### symmetry
if self.params['symmetry'] is not None:
xmippopts += " -sym " + self.params['symmetry'] + " "
if self.nproc > 1 and self.mpirun is not None:
### use multi-processor
apDisplay.printColor("Using " + str(self.nproc) + " processors!",
"green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_refine3d", die=True)
mpiruncmd = self.mpirun + " -np " + str(
self.nproc) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_refine3d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
apDisplay.printMsg("Reconstruction time: " +
apDisplay.timeString(time.time() - recontime))
def start(self):
self.iflag = 1
self.setIBLOW()
### get stack info
self.stackdata = apStack.getOnlyStackData(self.params['stackid'])
self.refinestackfile = os.path.join(self.stackdata['path']['path'],
self.stackdata['name'])
apImagicFile.setImagic4DHeader(self.refinestackfile)
### copy stack or start job file
if self.params['cluster'] is False:
#create alias to stack data
pass
if self.params['cluster'] is True:
self.setupMultiNode()
if self.params['reconstackid'] is not None:
self.reconstackdata = apStack.getOnlyStackData(
self.params['stackid'])
self.reconstackfile = os.path.join(self.stackdata['path']['path'],
self.stackdata['name'])
else:
self.reconstackfile = self.refinestackfile
### create initial model file
self.currentvol = os.path.basename(self.setupInitialModel())
### create parameter file
self.currentparam = os.path.basename(self.setupParticleParams())
apDisplay.printColor(
"Initial files:\n Stack: %s\n Volume: %s\n Params: %s\n" %
(os.path.basename(
self.refinestackfile), self.currentvol, self.currentparam),
"violet")
## run frealign for number for refinement cycles
for i in range(self.params['numiter']):
iternum = i + 1
if self.params['cluster'] is True:
self.multiNodeRun(iternum)
else:
self.singleNodeRun(iternum)
time.sleep(2)
### calculate FSC
#emancmd = 'proc3d %s %s fsc=fsc.eotest.%d' % (evenvol, oddvol, self.params['iter'])
#apEMAN.executeEmanCmd(emancmd, verbose=True)
if self.params['cluster'] is True:
self.prepareForCluster()
print "Done!"
def convertDefociToConvention(ctfvalues):
if debug is True:
apDisplay.printColor(
"Final params: def1: %.2e | def2: %.2e | angle: %.1f" %
(ctfvalues['defocus1'], ctfvalues['defocus2'],
ctfvalues['angle_astigmatism']), "cyan")
# amplitude contrast must be btw 0.0 and 0.5
# sometimes we get a slightly negative number from ACE1, see bug #2003
if abs(ctfvalues['amplitude_contrast']) < 0.005:
ctfvalues['amplitude_contrast'] = 0.0
# program specific corrections?
angle = ctfvalues['angle_astigmatism']
#by convention: abs(ctfvalues['defocus1']) < abs(ctfvalues['defocus2'])
if abs(ctfvalues['defocus1']) > abs(ctfvalues['defocus2']):
# incorrect, need to shift angle by 90 degrees
apDisplay.printWarning("|def1| > |def2|, flipping defocus axes")
defocus1 = ctfvalues['defocus2']
defocus2 = ctfvalues['defocus1']
angle += 90
else:
# correct, ratio > 1
defocus1 = ctfvalues['defocus1']
defocus2 = ctfvalues['defocus2']
if defocus1 < 0 and defocus2 < 0:
apDisplay.printWarning(
"Negative defocus values, taking absolute value")
defocus1 = abs(defocus1)
defocus2 = abs(defocus2)
# get angle within range -90 < angle <= 90
while angle > 90:
angle -= 180
while angle < -90:
angle += 180
if debug is True:
apDisplay.printColor(
"Final params: def1: %.2e | def2: %.2e | angle: %.1f" %
(defocus1, defocus2, angle), "cyan")
perdiff = abs(defocus1 - defocus2) / abs(defocus1 + defocus2)
print("Defocus Astig Percent Diff %.2f -- %.3e, %.3e" %
(perdiff * 100, defocus1, defocus2))
ctfvalues['defocus1'] = defocus1
ctfvalues['defocus2'] = defocus2
ctfvalues['angle_astigmatism'] = angle
return ctfvalues
def convertVolToSpider(self,
mrcvolfile=None,
modelid=None,
apix=None,
spivolfile=None):
"""
takes the mrc volume and creates a spider file ready for processing
"""
if modelid is not None:
initModelData = appiondata.ApInitialModelData.direct_query(modelid)
mrcvolfile = initModelData['path']['path'] + "/" + initModelData[
'name']
apix = initModelData['pixelsize']
stackapix = apStack.getStackPixelSizeFromStackId(
self.params['partstackid']) * self.params['bin']
stackboxsize = apStack.getStackBoxsize(
self.params['partstackid']) / self.params['bin']
print apix
print mrcvolfile
if not os.path.isfile(mrcvolfile):
apDisplay.printError("volfile does not exist: " + mrcvolfile)
### first high pass filter particles
#apDisplay.printMsg("pre-filtering particles")
#apix = apStack.getStackPixelSizeFromStackId(self.params['partstackid'])
#emancmd = ("proc3d "+mrcvolfile+" "+emanstackfile
# +" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
# +" inplace")
#apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc3d "
emancmd += mrcvolfile + " "
if spivolfile is None:
spivolfile = os.path.join(self.params['rundir'], "threed-0a.spi")
apFile.removeFile(spivolfile, warn=True)
emancmd += spivolfile + " "
emancmd += "scale=" + str(apix / stackapix) + " "
emancmd += "clip=" + str(stackboxsize) + "," + str(
stackboxsize) + "," + str(stackboxsize) + " "
emancmd += "spidersingle"
starttime = time.time()
apDisplay.printColor("Running spider volume conversion", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor(
"finished conversion in " +
apDisplay.timeString(time.time() - starttime), "cyan")
return spivolfile
def _startLoop(self, tiltseriesdata):
"""
initilizes several parameters for a new series
and checks if it is okay to start processing series
"""
#calc series left
self.stats['seriesleft'] = self.stats['seriescount'] - self.stats['count']
if self.params['background'] is False:
apDisplay.printColor( "\nStarting series %d ( skip:%d, remain:%d ) id:%d"
%(tiltseriesdata['number'], self.stats['skipcount'], self.stats['seriesleft'],
tiltseriesdata.dbid,),
"green")
#only if a series was processed last
if(self.stats['lastcount'] != self.stats['count']):
sys.stderr.write("\n")
self.stats['lastcount'] = self.stats['count']
self._checkMemLeak()
# skip if last image belong to the series doesn't exist:
imgtree = apDatabase.getImagesFromTiltSeries(tiltseriesdata,False)
imgpath = os.path.join(tiltseriesdata['session']['image path'], imgtree[0]['filename']+'.mrc')
if not os.path.isfile(imgpath):
apDisplay.printWarning(imgpath+" not found, skipping")
return False
# skip if there are some problem with the series
if self.__isShortTiltSeries(tiltseriesdata) or self.isBadTiltSeries(tiltseriesdata):
apDisplay.printWarning("Series %d is not good enough for processing, skipping" % (tiltseriesdata['number']))
seriesname = "series%3d" % (tiltseriesdata['number'])
self._writeDoneDict(seriesname)
self.stats['count'] += 1
return False
# check to see if series has already been processed
if self._alreadyProcessed(tiltseriesdata):
return False
self.stats['waittime'] = 0
if self.reprocessSeries(tiltseriesdata) is True:
if self.params['background'] is True:
sys.stderr.write(",")
else:
"""apDisplay.printMsg("reprocessing series %d" % (tiltseriesdata['number']))"""
else:
if self.params['background'] is True:
sys.stderr.write(".")
else:
"""apDisplay.printMsg("processing series %d" % (tiltseriesdata['number']))"""
return True
def convertStackToSpider(self, emanstackfile):
"""
takes the stack file and creates a spider file ready for processing
"""
if not os.path.isfile(emanstackfile):
apDisplay.printError("stackfile does not exist: " + emanstackfile)
tempstack = os.path.join(self.params['rundir'],
"filter" + self.timestamp + ".hed")
### first high pass filter particles
apDisplay.printMsg("pre-filtering particles")
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
boxsize = self.getBoxSize()
emancmd = ("proc2d " + emanstackfile + " " + tempstack + " apix=" +
str(apix) + " ")
if self.params['highpasspart'] is not None and self.params[
'highpasspart'] > 0:
emancmd += "hp=" + str(self.params['highpasspart']) + " "
if self.params[
'lowpasspart'] is not None and self.params['lowpasspart'] > 0:
emancmd += "lp=" + str(self.params['lowpasspart']) + " "
if self.params['tiltbin'] > 1:
clipsize = boxsize * self.params['tiltbin']
emancmd += " shrink=%d clip=%d,%d " % (self.params['tiltbin'],
clipsize, clipsize)
apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc2d "
emancmd += tempstack + " "
spiderstack = os.path.join(self.params['rundir'],
"rctstack" + self.timestamp + ".spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack + " "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor(
"Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
time.sleep(1) # wait a sec, for things to finish
apDisplay.printColor(
"finished eman in " +
apDisplay.timeString(time.time() - starttime), "cyan")
apFile.removeStack(tempstack, warn=False)
apFile.removeStack(emanstackfile, warn=False)
if not os.path.isfile(spiderstack):
apDisplay.printError("Failed to create a spider stack")
return spiderstack
def start(self):
### database entry parameters
package_table = 'ApFrealignIterData|frealignParams'
### set projection-matching path
self.projmatchpath = os.path.abspath(
os.path.join(self.params['rundir'], "recon"))
### determine which iterations to upload
lastiter = self.findLastCompletedIteration()
uploadIterations = self.verifyUploadIterations(lastiter)
for iteration in uploadIterations:
apDisplay.printColor("uploading iteration %d" % iteration, "cyan")
package_database_object = self.instantiateProjMatchParamsData(
iteration)
if not self.params['euleronly']:
### move FSC file to results directory
self.FSCExists = self.convertFSCFileForIteration(iteration)
### create a text file with particle information
self.createParticleDataFile(iteration, package_database_object)
if not self.params['euleronly']:
### create mrc file of map for iteration and reference number
oldvol = os.path.join(self.projmatchpath,
"threed.%03da.mrc" % iteration)
newvol = os.path.join(
self.resultspath, "recon_%s_it%.3d_vol001.mrc" %
(self.params['timestamp'], iteration))
apFile.safeCopy(oldvol, newvol)
### make chimera snapshot of volume
self.createChimeraVolumeSnapshot(newvol, iteration)
### instantiate database objects
self.insertRefinementRunData(iteration)
self.insertRefinementIterationData(iteration, package_table,
package_database_object)
### make symlink only after successful insertion
if not self.params['euleronly'] and os.path.isfile(newvol):
if os.path.isfile(oldvol):
apFile.removeFile(oldvol, True)
try:
os.symlink(newvol, oldvol)
except IOError, e:
print e
def createJobFiles(self):
self.goldStandardResolution()
for i in range(self.params['numiter']):
for side in ('left', 'right'):
apDisplay.printColor("iteration %02d, side %s" % (i + 1, side),
"cyan")
stackFile = os.path.join("..", self.splitStacks[side])
oldprefix = "%s.iter%02d" % (side, i)
newprefix = "%s.iter%02d" % (side, i + 1)
self.createIterationJobRefineFiles(oldprefix, newprefix,
stackFile)
self.createReconstructionJobs(newprefix, stackFile)
def insertAlignParticlesIntoDatabase(self):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for ref in self.classD:
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = ref
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = 10
# if ref in self.resdict:
# refq['ssnr_resolution'] = self.resdict[ref]
### setup particle info
for partnum in self.classD[ref]:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(partnum)
alignpartq['alignstack'] = self.alignstackdata
### particle numbering starts with 1 in SIMPLE
stackpartdata = apStack.getStackParticle(
self.runparams['stackid'], int(partnum))
alignpartq['stackpart'] = stackpartdata
alignpartq['ref'] = refq
try:
alignpartq['xshift'] = self.alignD[partnum - 1]['shx']
except:
pass
try:
alignpartq['yshift'] = self.alignD[partnum - 1]['shy']
except:
pass
try:
alignpartq['mirror'] = self.alignD[partnum - 1]['mirror']
except:
pass
alignpartq['rotation'] = self.alignD[partnum - 1]['rot']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " +
str(self.params['numpart']) + " particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def insertClusterStack(self,
classavg=None,
classvar=None,
numclass=None,
insert=False):
clusterstackq = appiondata.ApClusteringStackData()
clusterstackq['avg_imagicfile'] = classavg + ".hed"
clusterstackq['var_imagicfile'] = classvar + ".hed"
clusterstackq['num_classes'] = numclass
clusterstackq['clusterrun'] = self.clusterrun
clusterstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
clusterstackq['hidden'] = False
imagicfile = os.path.join(self.params['rundir'],
clusterstackq['avg_imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find average stack file: " +
imagicfile)
imagicfile = os.path.join(self.params['rundir'],
clusterstackq['var_imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find variance stack file: " +
imagicfile)
apDisplay.printMsg("inserting clustering stack into database")
if insert is True:
clusterstackq.insert()
### particle class data
apDisplay.printColor(
"Inserting particle classification data, please wait", "cyan")
for i in range(numclass):
classnum = i + 1
classdocfile = os.path.join(
self.params['rundir'],
"cluster/classdoc_%s_%04d.spi" % (self.timestamp, classnum))
partlist = self.readClassDocFile(classdocfile)
sys.stderr.write(".")
for partnum in partlist:
alignpartdata = self.getAlignParticleData(partnum)
cpartq = appiondata.ApClusteringParticleData()
cpartq['clusterstack'] = clusterstackq
cpartq['alignparticle'] = alignpartdata
cpartq['partnum'] = partnum
cpartq['refnum'] = classnum
cpartq['clusterreference'] = None
# actual parameters
if insert is True:
cpartq.insert()
return
def setFileName(self, unique=False):
if self.params['name'] is None:
### assign provided name
if self.params['pdbid'] is not None:
basename = "pdb%s-%s"%(self.params['pdbid'], self.timestamp)
else:
filebase = os.path.splitext(os.path.basename(self.params['pdbfile']))[0]
basename = "%s-%s"%(filebase, self.timestamp)
else:
### clean up provided name
basename = os.path.splitext(os.path.basename(self.params['name']))[0]
self.params['name'] = os.path.join(self.params['rundir'], basename)
apDisplay.printColor("Naming PDB model: "+self.params['name'], "cyan")
return
