def getParticleTiltDefocus(ctfdata, imgdata, NX, NY):
### calculate defocus at given position
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
CX = dimx / 2
CY = dimy / 2
if ctfdata['tilt_axis_angle'] is not None:
N1 = -1.0 * math.sin(math.radians(ctfdata['tilt_axis_angle']))
N2 = math.cos(math.radians(ctfdata['tilt_axis_angle']))
else:
N1 = 0.0
N2 = 1.0
PSIZE = apDatabase.getPixelSize(imgdata)
### High tension on CM is given in kv instead of v so do not divide by 1000 in that case
if imgdata['scope']['tem']['name'] == "CM":
voltage = imgdata['scope']['high tension']
else:
voltage = (imgdata['scope']['high tension']) / 1000
# flip Y axis due to reversal of y coordinates
NY = dimy - NY
DX = CX - NX
DY = CY - NY
DF = (N1 * DX + N2 * DY) * PSIZE * math.tan(
math.radians(ctfdata['tilt_angle']))
#print "using tilt axis: %.02f, angle: %.02f"%(ctfdata['tilt_axis_angle'],ctfdata['tilt_angle'])
#print "df1: %.2f, df2: %.2f, offset is %.2f"%(ctfdata['defocus1']*-1e10,ctfdata['defocus2']*-1e10,DF)
DFL1 = abs(ctfdata['defocus1']) * 1.0e10 + DF
DFL2 = abs(ctfdata['defocus2']) * 1.0e10 + DF
return DFL1, DFL2
def runAceCorrect(imgdict,params):
imgname = imgdict['filename']
imgpath = os.path.join(imgdict['session']['image path'], imgname+'.mrc')
voltage = (imgdict['scope']['high tension'])
apix = apDatabase.getPixelSize(imgdict)
ctfvalues = ctfdb.getBestCtfByResolution(imgdata)
conf = ctfdb.calculateConfidenceScore(bestctfvalue)
ctdimname = imgname
ctdimpath = os.path.join(params['rundir'],ctdimname)
print "Corrected Image written to " + ctdimpath
#pdb.set_trace()
acecorrectcommand=("ctfcorrect1('%s', '%s', '%.32f', '%.32f', '%f', '%f', '%f');" % \
(imgpath, ctdimpath, ctfvalues['defocus1'], ctfvalues['defocus2'], -ctfvalues['angle_astigmatism'], voltage, apix))
print acecorrectcommand
try:
matlab = pymat.open("matlab -nosplash")
except:
apDisplay.environmentError()
raise
pymat.eval(matlab, acecorrectcommand)
pymat.close(matlab)
return
def processParticles(self, imgdata, partdatas, shiftdata):
self.shortname = apDisplay.short(imgdata['filename'])
### if only selected points along helix,
### fill in points with helical step
if self.params['helicalstep']:
apix = apDatabase.getPixelSize(imgdata)
partdatas = self.fillWithHelicalStep(partdatas, apix)
### run batchboxer
self.boxedpartdatas, self.imgstackfile, self.partmeantree = self.boxParticlesFromImage(imgdata, partdatas, shiftdata)
if self.boxedpartdatas is None:
self.stats['lastpeaks'] = 0
apDisplay.printWarning("no particles were boxed from "+self.shortname+"\n")
self.badprocess = True
return None
self.stats['lastpeaks'] = len(self.boxedpartdatas)
apDisplay.printMsg("do not break function now otherwise it will corrupt stack")
#time.sleep(1.0)
### merge image particles into big stack
totalpart = self.mergeImageStackIntoBigStack(self.imgstackfile, imgdata)
### create a stack average every so often
if self.stats['lastpeaks'] > 0:
totalPartices = self.existingParticleNumber+self.stats['peaksum']+self.stats['lastpeaks']
logpeaks = math.log(totalPartices)
if logpeaks > self.logpeaks:
self.logpeaks = math.ceil(logpeaks)
numpeaks = math.ceil(math.exp(self.logpeaks))
apDisplay.printMsg("writing averaging stack, next average at %d particles"%(numpeaks))
mrc.write(self.summedParticles/float(totalPartices), "average.mrc")
return totalpart
def getFrealignStyleParticleMagnification(stackpdata,refinestack_bin): stackbin = apStack.getStackBinningFromStackId(stackpdata['stack'].dbid) totalbin = stackbin * refinestack_bin imgdata = stackpdata['particle']['image'] apix = apDatabase.getPixelSize(imgdata) # in angstroms camera_pixel = imgdata['camera']['pixel size']['x']*imgdata['camera']['binning']['x'] # in meters return camera_pixel / (apix * 1e-10 * totalbin)
def processAndSaveFFT(self, imgdata, fftpath):
if os.path.isfile(fftpath):
print "FFT file found"
if fftpath in self.freqdict.keys():
print "Freq found"
return False
print "Freq not found"
print "creating FFT file: ", fftpath
### downsize and filter leginon image
if self.params['uncorrected']:
imgarray = imagefilter.correctImage(imgdata, params)
else:
imgarray = imgdata['image']
### calculate power spectra
apix = apDatabase.getPixelSize(imgdata)
fftarray, freq = ctfpower.power(imgarray, apix, mask_radius=0.5, fieldsize=self.params['fieldsize'])
#fftarray = imagefun.power(fftarray, mask_radius=1)
fftarray = ndimage.median_filter(fftarray, 2)
## preform a rotational average and remove peaks
rotfftarray = ctftools.rotationalAverage2D(fftarray)
stdev = rotfftarray.std()
rotplus = rotfftarray + stdev*4
fftarray = numpy.where(fftarray > rotplus, rotfftarray, fftarray)
### save to jpeg
self.freqdict[fftpath] = freq
mrc.write(fftarray, fftpath)
self.saveFreqFile()
return True
def checkPixelSize(self):
# make sure that images all have same pixel size:
# first get pixel size of first image:
self.params['apix'] = None
for imgdata in self.imgtree:
# get pixel size
imgname = imgdata['filename']
if imgname in self.donedict:
continue
if self.params['apix'] is None:
self.params['apix'] = apDatabase.getPixelSize(imgdata)
apDisplay.printMsg("Stack pixelsize = %.3f A"%(self.params['apix']))
if apDatabase.getPixelSize(imgdata) != self.params['apix']:
apDisplay.printMsg("Image pixelsize %.3f A != Stack pixelsize %.3f A"%(apDatabase.getPixelSize(imgdata), self.params['apix']))
apDisplay.printMsg("Problem image name: %s"%(apDisplay.short(imgdata['filename'])))
apDisplay.printError("This particle selection run contains images of varying pixelsizes, a stack cannot be created")
def runAceCorrect(imgdict,params):
imgname = imgdict['filename']
imgpath = os.path.join(imgdict['session']['image path'], imgname+'.mrc')
voltage = (imgdict['scope']['high tension'])
apix = apDatabase.getPixelSize(imgdict)
ctfvalues, conf = ctfdb.getBestCtfValueForImage(imgdict)
ctdimname = imgname
ctdimpath = os.path.join(params['rundir'],ctdimname)
print "Corrected Image written to " + ctdimpath
#pdb.set_trace()
acecorrectcommand=("ctfcorrect1('%s', '%s', '%.32f', '%.32f', '%f', '%f', '%f');" % \
(imgpath, ctdimpath, ctfvalues['defocus1'], ctfvalues['defocus2'], -ctfvalues['angle_astigmatism'], voltage, apix))
print acecorrectcommand
try:
matlab = pymat.open("matlab -nosplash")
except:
apDisplay.environmentError()
raise
pymat.eval(matlab, acecorrectcommand)
pymat.close(matlab)
return
def processImage(self, imgdata):
self.ctfvalues = {}
# RESTART MATLAB EVERY 500 IMAGES OR IT RUNS OUT OF MEMORY
if self.stats['count'] % 500 == 0:
apDisplay.printWarning("processed 500 images. restarting matlab...")
pymat.close(self.matlab)
time.sleep(5)
self.matlab = pymat.open()
scopeparams = {
'kv': imgdata['scope']['high tension']/1000,
'apix': apDatabase.getPixelSize(imgdata),
'cs': self.params['cs'],
'tempdir': self.params['tempdir'],
}
### Scott's hack for FSU CM
### For CM high tension is given in kv instead of v
if imgdata['scope']['tem']['name'] == "CM":
scopeparams['kv'] = imgdata['scope']['high tension']
apMatlab.setScopeParams(self.matlab, scopeparams)
### RUN ACE
self.ctfvalues = {}
acevalues = apMatlab.runAce(self.matlab, imgdata, self.params)
#check if ACE was successful
if acevalues[0] == -1:
self.badprocess = True
return False
acevaluelist = ('defocus1','defocus2','defocusinit','amplitude_contrast','angle_astigmatism',
'noise1','noise2','noise3','noise4','envelope1','envelope2','envelope3','envelope4',
'lowercutoff','uppercutoff','snr','confidence','confidence_d')
for i in range(len(acevaluelist)):
self.ctfvalues[ acevaluelist[i] ] = acevalues[i]
### override the astig angle to be in degrees
self.ctfvalues['angle_astigmatism'] = - math.degrees(self.ctfvalues['angle_astigmatism'])
self.ctfvalues['mat_file'] = imgdata['filename']+".mrc.mat"
self.ctfvalues['cs'] = scopeparams['cs']
self.ctfvalues['volts'] = scopeparams['kv']*1000.0
### RENAME/MOVE OUTPUT FILES
imfile1 = os.path.join(self.params['tempdir'], "im1.png")
imfile2 = os.path.join(self.params['tempdir'], "im2.png")
opimfile1 = imgdata['filename']+".mrc1.png"
opimfile2 = imgdata['filename']+".mrc2.png"
opimfilepath1 = os.path.join(self.powerspecdir, opimfile1)
opimfilepath2 = os.path.join(self.powerspecdir, opimfile2)
if os.path.isfile(imfile1):
shutil.copyfile(imfile1, opimfilepath1)
else:
apDisplay.printWarning("imfile1 is missing, %s"%(imfile1))
if os.path.isfile(imfile2):
shutil.copyfile(imfile2, opimfilepath2)
else:
apDisplay.printWarning("imfile2 is missing, %s"%(imfile2))
self.ctfvalues['graph1'] = opimfilepath1
self.ctfvalues['graph2'] = opimfilepath2
def getStackPixelSizeFromStackId(stackId, msg=True):
"""
For a given stack id return stack apix
Not tested on defocal pairs
"""
stackdata = getOnlyStackData(stackId, msg=msg)
if stackdata['pixelsize'] is not None:
### Quicker method
stackapix = stackdata['pixelsize'] * 1e10
if msg is True:
apDisplay.printMsg("Stack " + str(stackId) + " pixel size: " +
str(round(stackapix, 3)))
return stackapix
apDisplay.printWarning(
"Getting stack pixel size from leginon DB, not tested on defocal pairs"
)
stackpart = getOneParticleFromStackId(stackId, msg=msg)
imgapix = apDatabase.getPixelSize(stackpart['particle']['image'])
stackbin = getStackBinningFromStackId(stackId)
stackapix = imgapix * stackbin
if msg is True:
apDisplay.printMsg("Stack " + str(stackId) + " pixel size: " +
str(round(stackapix, 3)))
return stackapix
def phaseFlipWholeImage(self, inimgpath, imgdata):
outimgpath = os.path.join(self.params['rundir'], self.shortname+"-ctfcorrect.dwn.mrc")
### High tension on CM is given in kv instead of v so do not divide by 1000 in that case
if imgdata['scope']['tem']['name'] == "CM":
voltage = imgdata['scope']['high tension']
else:
voltage = (imgdata['scope']['high tension'])/1000
apix = apDatabase.getPixelSize(imgdata)
defocus, ampconst = self.getDefocusAmpConstForImage(imgdata, True)
defocus *= 1.0e6
self.checkDefocus(defocus, self.shortname)
### get all CTF parameters, we also need to get the CS value from the database
ctfdata = self.getBestCtfValue(imgdata, False)
#ampconst = ctfdata['amplitude_contrast'] ### we could use this too
# find cs
cs = self.getCS(ctfdata)
parmstr = ("parm=%f,200,1,%.3f,0,17.4,9,1.53,%i,%.1f,%f" %(defocus, ampconst, voltage, cs, apix))
emancmd = ("applyctf %s %s %s setparm flipphase" % (inimgpath, outimgpath, parmstr))
apDisplay.printMsg("phaseflipping entire micrograph with defocus "+str(round(defocus,3))+" microns")
apEMAN.executeEmanCmd(emancmd, showcmd = True)
return outimgpath
def getParticleTiltDefocus(ctfdata,imgdata,NX,NY):
### calculate defocus at given position
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
CX = dimx/2
CY = dimy/2
if ctfdata['tilt_axis_angle'] is not None:
N1 = -1.0 * math.sin( math.radians(ctfdata['tilt_axis_angle']) )
N2 = math.cos( math.radians(ctfdata['tilt_axis_angle']) )
else:
N1 = 0.0
N2 = 1.0
PSIZE = apDatabase.getPixelSize(imgdata)
### High tension on CM is given in kv instead of v so do not divide by 1000 in that case
if imgdata['scope']['tem']['name'] == "CM":
voltage = imgdata['scope']['high tension']
else:
voltage = (imgdata['scope']['high tension'])/1000
# flip Y axis due to reversal of y coordinates
NY = dimy-NY
DX = CX - NX
DY = CY - NY
DF = (N1*DX + N2*DY) * PSIZE * math.tan( math.radians(ctfdata['tilt_angle']) )
#print "using tilt axis: %.02f, angle: %.02f"%(ctfdata['tilt_axis_angle'],ctfdata['tilt_angle'])
#print "df1: %.2f, df2: %.2f, offset is %.2f"%(ctfdata['defocus1']*-1e10,ctfdata['defocus2']*-1e10,DF)
DFL1 = abs(ctfdata['defocus1'])*1.0e10 + DF
DFL2 = abs(ctfdata['defocus2'])*1.0e10 + DF
return DFL1,DFL2
def preLoopFunctions(self):
if len(self.imgtree) > 0:
self.params['apix'] = apDatabase.getPixelSize(self.imgtree[0])
# CREATES TEMPLATES
# SETS params['templatelist'] AND self.params['templateapix']
apTemplate.getTemplates(self.params)
apDisplay.printColor("Template list: "+str(self.params['templatelist']), "cyan")
self.checkPreviousTemplateRun()
def preLoopFunctions(self):
if len(self.imgtree) > 0:
self.params['apix'] = apDatabase.getPixelSize(self.imgtree[0])
# CREATES TEMPLATES
# SETS params['templatelist'] AND self.params['templateapix']
apTemplate.getTemplates(self.params)
apDisplay.printColor(
"Template list: " + str(self.params['templatelist']), "cyan")
self.checkPreviousTemplateRun()
def getImagePixelSizeFromContourId(contourid):
# This returns pixelsize in Angstrom
q = appiondata.ApSelectionRunData()
rundata = q.direct_query(contourid)
q = appiondata.ApContourData(selectionrun=rundata)
r = q.query(results=1)
lastimagedata = r[0]['image']
apix = apDatabase.getPixelSize(lastimagedata)
return apix
def phaseFlipParticles(self, imgdata, imgstackfile):
apDisplay.printMsg("Applying CTF to particles")
ctfimgstackfile = os.path.join(self.params['rundir'], self.shortname+"-ctf.hed")
### High tension on CM is given in kv instead of v so do not divide by 1000 in that case
if imgdata['scope']['tem']['name'] == "CM":
voltage = imgdata['scope']['high tension']
else:
voltage = (imgdata['scope']['high tension'])/1000
apix = apDatabase.getPixelSize(imgdata)
### get the adjusted defocus value for no astigmatism
defocus, ampconst = self.getDefocusAmpConstForImage(imgdata, True)
defocus *= 1.0e6
### check to make sure defocus is a reasonable value for applyctf
self.checkDefocus(defocus, self.shortname)
### get all CTF parameters, we also need to get the CS value from the database
ctfdata = self.getBestCtfValue(imgdata, False)
#ampconst = ctfdata['amplitude_contrast'] ### we could use this too
# find cs
cs = self.getCS(ctfdata)
"""
// from EMAN1 source code: EMAN/src/eman/libEM/EMTypes.h
and EMAN/src/eman/libEM/EMDataA.C
struct CTFParam {
float defocus; // in microns, negative underfocus
float bfactor; // not needed for phaseflip, envelope function width (Pi/2 * Wg)^2
float amplitude; // not needed for phaseflip,
ctf amplitude, mutliplied times the entire equation
float ampcont; // number from 0 to 1, sqrt(1 - a^2) format
float noise1/exps4; // not needed for phaseflip, noise exponential decay amplitude
float noise2; // not needed for phaseflip, width
float noise3; // not needed for phaseflip, noise gaussian amplitude
float noise4; // not needed for phaseflip, noide gaussian width
float voltage; // in kilovolts
float cs; // in millimeters
float apix; // in Angstroms per pixel
};
noise follows noise3*exp[ -1*(pi/2*noise4*x0)^2 - x0*noise2 - sqrt(fabs(x0))*noise1]
"""
parmstr = ("parm=%f,200,1,%.3f,0,17.4,9,1.53,%i,%.1f,%f"
%(defocus, ampconst, voltage, cs, apix))
emancmd = ("applyctf %s %s %s setparm flipphase"
%(imgstackfile, ctfimgstackfile, parmstr))
apDisplay.printMsg("phaseflipping particles with defocus "+str(round(defocus,3))+" microns")
apEMAN.executeEmanCmd(emancmd, showcmd = True)
return ctfimgstackfile
def _getAllImages(self):
startt = time.time()
if self.params['mrcnames'] is not None:
mrcfileroot = self.params['mrcnames'].split(",")
if self.params['sessionname'] is not None:
self.imgtree = apDatabase.getSpecificImagesFromSession(
mrcfileroot, self.params['sessionname'])
else:
self.imgtree = apDatabase.getSpecificImagesFromDB(mrcfileroot)
elif self.params['sessionname'] is not None:
if self.params['preset'] is not None:
self.imgtree = apDatabase.getImagesFromDB(
self.params['sessionname'], self.params['preset'])
else:
self.imgtree = apDatabase.getAllImagesFromDB(
self.params['sessionname'])
else:
if self.params['mrcnames'] is not None:
apDisplay.printMsg("MRC List: " +
str(len(self.params['mrcnames'])) + " : " +
str(self.params['mrcnames']))
apDisplay.printMsg("Session: " + str(self.params['sessionname']) +
" : " + str(self.params['preset']))
apDisplay.printError("no files specified")
precount = len(self.imgtree)
apDisplay.printMsg("Found " + str(precount) + " images in " +
apDisplay.timeString(time.time() - startt))
### REMOVE PROCESSED IMAGES
apDisplay.printMsg("Remove processed images")
self._removeProcessedImages()
### SET IMAGE ORDER
if self.params['shuffle'] is True:
self.imgtree = self._shuffleTree(self.imgtree)
apDisplay.printMsg("Process images shuffled")
elif self.params['reverse'] is True:
apDisplay.printMsg("Process images new to old")
else:
# by default images are new to old
apDisplay.printMsg("Process images old to new")
self.imgtree.sort(self._reverseSortImgTree)
### LIMIT NUMBER
if self.params['limit'] is not None:
lim = self.params['limit']
if len(self.imgtree) > lim:
apDisplay.printMsg("Limiting number of images to " + str(lim))
self.imgtree = self.imgtree[:lim]
if len(self.imgtree) > 0:
self.params['apix'] = apDatabase.getPixelSize(self.imgtree[0])
self.stats['imagecount'] = len(self.imgtree)
def checkPixelSize(self):
# make sure that images all have same pixel size:
# first get pixel size of first image:
self.params['apix'] = None
for imgdata in self.imgtree:
# get pixel size
imgname = imgdata['filename']
if imgname in self.donedict:
continue
if self.params['apix'] is None:
self.params['apix'] = apDatabase.getPixelSize(imgdata)
apDisplay.printMsg("Stack pixelsize = %.3f A" %
(self.params['apix']))
if apDatabase.getPixelSize(imgdata) != self.params['apix']:
apDisplay.printMsg(
"Image pixelsize %.3f A != Stack pixelsize %.3f A" %
(apDatabase.getPixelSize(imgdata), self.params['apix']))
apDisplay.printMsg("Problem image name: %s" %
(apDisplay.short(imgdata['filename'])))
apDisplay.printError(
"This particle selection run contains images of varying pixelsizes, a stack cannot be created"
)
def getInstrumentParams(self):
self.stackpix = apStack.getStackPixelSizeFromStackId(self.stackid, True) * 1e-10
stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum+1, msg=True)
pdata = stackparticledata['particle']
self.imgpix = apDatabase.getPixelSize(pdata['image'])*1e-10
scopedata = pdata['image']['scope']
self.tem = scopedata['tem']
self.cam = pdata['image']['camera']['ccdcamera']
self.ht = scopedata['high tension']
self.Cs = 2e-3
self.mag = scopedata['magnification']
self.camsize = {'x':4096,'y':4096}
self.beamdiameter = 1e-6
self.c2diameter = 1.0e-4
self.wavelength = fftfun.getElectronWavelength(self.ht)
def uploadTomo(params):
if not params['commit']:
apDisplay.printWarning("not commiting tomogram to database")
return
apDisplay.printMsg("Commiting tomogram to database")
sessiondata = apDatabase.getSessionDataFromSessionName(params['sessionname'])
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(params['tiltseriesnumber'],sessiondata)
runname = params['runname']
name = params['name']
if params['full']:
fullbin = params['bin']
else:
fullbin = 1
subbin = params['bin']
alignrun = insertTomoAlignmentRun(sessiondata,None,None,None,None,fullbin,runname,params['aligndir'],'manual alignment from upload')
# only tilt series in one alignrun for now
insertTiltsInAlignRun(alignrun, tiltdata,None,True)
alignerdata = insertAlignerParams(alignrun,params)
firstimagedata = getFirstImage(tiltdata)
path = os.path.abspath(params['rundir'])
description = params['description']
if params['full']:
thickness = params['shape'][0] * fullbin
uploadfile = params['zprojfile']
projectimagedata = uploadZProjection(runname,firstimagedata,uploadfile)
fullrundata = insertFullTomoRun(sessiondata,path,runname,'upload')
return insertFullTomogram(sessiondata,tiltdata,alignerdata,fullrundata,name,description,projectimagedata,thickness,None,fullbin,[])
else:
projectimagedata = None
fulltomopath = params['rundir'].replace('/'+params['volume'],'')
dummyname = 'dummy'
dummydescription = 'fake full tomogram for subtomogram upload'
thickness = params['shape'][0] * subbin
fullrundata = insertFullTomoRun(sessiondata,fulltomopath,runname,'upload')
fulltomogram = insertFullTomogram(sessiondata,tiltdata,alignerdata,fullrundata,dummyname,dummydescription,projectimagedata,thickness,None,fullbin,[])
apix = apDatabase.getPixelSize(firstimagedata)
tomoq = appiondata.ApTomogramData()
tomoq['session'] = sessiondata
tomoq['tiltseries'] = tiltdata
results = tomoq.query()
index = len(results)+1
pixelsize = 1e-10 * apix * subbin
runname = params['volume']
shape = map((lambda x: x * subbin), params['shape'])
dimension = {'x':shape[2],'y':shape[1], 'z':shape[0]}
subtomorundata = insertSubTomoRun(sessiondata,
None,None,runname,params['invert'],subbin)
return insertSubTomogram(fulltomogram,subtomorundata,None,0,dimension,path,name,index,pixelsize,description)
def preLoopFunctions(self):
if len(self.imgtree) > 0:
self.params['apix'] = apDatabase.getPixelSize(self.imgtree[0])
# CREATES TEMPLATES
# SETS params['templatelist'] AND self.params['templateapix']
apTemplate.getTemplates(self.params)
apDisplay.printColor("Template list: "+str(self.params['templatelist']), "cyan")
self.checkPreviousTemplateRun()
# convert templates to single mrc stack for signature
tstack="templatestack.mrc"
twods=[]
for t in self.params['templatelist']:
twods.append(mrc.read(t))
imgar=numpy.array(twods)
mrc.write(imgar, tstack)
self.params['templatename']=tstack
def _getAllImages(self):
startt = time.time()
if self.params['mrcnames'] is not None:
mrcfileroot = self.params['mrcnames'].split(",")
if self.params['sessionname'] is not None:
self.imgtree = apDatabase.getSpecificImagesFromSession(mrcfileroot, self.params['sessionname'])
else:
self.imgtree = apDatabase.getSpecificImagesFromDB(mrcfileroot)
elif self.params['sessionname'] is not None:
if self.params['preset'] is not None:
self.imgtree = apDatabase.getImagesFromDB(self.params['sessionname'], self.params['preset'])
else:
self.imgtree = apDatabase.getAllImagesFromDB(self.params['sessionname'])
else:
if self.params['mrcnames'] is not None:
apDisplay.printMsg("MRC List: "+str(len(self.params['mrcnames']))+" : "+str(self.params['mrcnames']))
apDisplay.printMsg("Session: "+str(self.params['sessionname'])+" : "+str(self.params['preset']))
apDisplay.printError("no files specified")
precount = len(self.imgtree)
apDisplay.printMsg("Found "+str(precount)+" images in "+apDisplay.timeString(time.time()-startt))
### REMOVE PROCESSED IMAGES
apDisplay.printMsg("Remove processed images")
self._removeProcessedImages()
### SET IMAGE ORDER
if self.params['shuffle'] is True:
self.imgtree = self._shuffleTree(self.imgtree)
apDisplay.printMsg("Process images shuffled")
elif self.params['reverse'] is True:
apDisplay.printMsg("Process images new to old")
else:
# by default images are new to old
apDisplay.printMsg("Process images old to new")
self.imgtree.sort(self._reverseSortImgTree)
### LIMIT NUMBER
if self.params['limit'] is not None:
lim = self.params['limit']
if len(self.imgtree) > lim:
apDisplay.printMsg("Limiting number of images to "+str(lim))
self.imgtree = self.imgtree[:lim]
if len(self.imgtree) > 0:
self.params['apix'] = apDatabase.getPixelSize(self.imgtree[0])
self.stats['imagecount'] = len(self.imgtree)
def postProcessParticleStack(self, imgdata, imgstackfile, boxedpartdatas, parttree_length):
### phase flipping
t0 = time.time()
if self.params['phaseflipped'] is True:
if self.params['fliptype'] == 'emantilt':
### ctf correct individual particles with tilt using Eman
imgstackfile = self.tiltPhaseFlipParticles(imgdata, imgstackfile, boxedpartdatas)
self.ctftimes.append(time.time()-t0)
elif self.params['fliptype'] == 'emanpart':
### ctf correct individual particles using Eman
imgstackfile = self.phaseFlipParticles(imgdata, imgstackfile)
self.ctftimes.append(time.time()-t0)
else:
apDisplay.printMsg("phase flipped whole image already")
else:
apDisplay.printMsg("not phase flipping particles")
numpart = apFile.numImagesInStack(imgstackfile)
apDisplay.printMsg(str(numpart)+" particles were boxed out from "+self.shortname)
if parttree_length != numpart:
apDisplay.printError("There is a mismatch in the number of particles expected and that were boxed")
### rectangular box masking
if self.params['boxmask'] is not None:
# create a stack of rectangular masks, which will be applied to
# the particles as they are added to the final stack
pixsz = apDatabase.getPixelSize(imgdata)*self.params['bin']
boxsz = self.boxsize/self.params['bin']
bxmask = int(self.params['bxmask']/pixsz)
bymask = int(self.params['bymask']/pixsz)
if self.params['bimask'] > 0:
bimask = int(self.params['bimask']/pixsz)
else: bimask = None
falloff = self.params['falloff']/pixsz
# adjust masks for falloff
bxmask -= falloff/2
bymask = (bymask/2)-(falloff/2)
self.params['boxmaskf'] = os.path.splitext(imgstackfile)[0]+"-mask.hed"
apBoxer.boxMaskStack(self.params['boxmaskf'], boxedpartdatas, boxsz,
bxmask, bymask, falloff, bimask, norotate=self.params['rotate'])
return imgstackfile
def function(self,rundata, imgdata, binnedimgarray):
self.params['apix'] = apDatabase.getPixelSize(imgdata)
# TODO: replace the call below with similar code from apCrud.py
# regions,maskarray = apCrud.makeMask(self.params, binnedimgarray)
regioninfos = []
test = params["test"] # test mode flag
lognumber = 0
testlog = [test,lognumber,filelog]
mask = Image.open()
labeled_regions,clabels = nd.label(mask)
regioninfos,testlog = getLabeledInfo(originalimage,Henrybitmap,labeled_regions,range(1,clabels+1),False,regioninfos,testlog)
regionTree = self.getResults(rundata, imgdata, regions)
return {'region':regionTree,'mask':maskarray}
def getInstrumentParams(self):
self.stackpix = apStack.getStackPixelSizeFromStackId(
self.stackid, True) * 1e-10
stackparticledata = apStack.getOneParticleFromStackId(self.stackid,
self.imgnum + 1,
msg=True)
pdata = stackparticledata['particle']
self.imgpix = apDatabase.getPixelSize(pdata['image']) * 1e-10
scopedata = pdata['image']['scope']
self.tem = scopedata['tem']
self.cam = pdata['image']['camera']['ccdcamera']
self.ht = scopedata['high tension']
self.Cs = 2e-3
self.mag = scopedata['magnification']
self.camsize = {'x': 4096, 'y': 4096}
self.beamdiameter = 1e-6
self.c2diameter = 1.0e-4
self.wavelength = fftfun.getElectronWavelength(self.ht)
def preLoopFunctions(self):
if len(self.imgtree) > 0:
self.params['apix'] = apDatabase.getPixelSize(self.imgtree[0])
# CREATES TEMPLATES
# SETS params['templatelist'] AND self.params['templateapix']
apTemplate.getTemplates(self.params)
apDisplay.printColor(
"Template list: " + str(self.params['templatelist']), "cyan")
self.checkPreviousTemplateRun()
# convert templates to single mrc stack for signature
tstack = "templatestack.mrc"
twods = []
for t in self.params['templatelist']:
twods.append(mrc.read(t))
imgar = numpy.array(twods)
mrc.write(imgar, tstack)
self.params['templatename'] = tstack
def function(self, rundata, imgdata, binnedimgarray):
self.params['apix'] = apDatabase.getPixelSize(imgdata)
# TODO: replace the call below with similar code from apCrud.py
# regions,maskarray = apCrud.makeMask(self.params, binnedimgarray)
regioninfos = []
test = params["test"] # test mode flag
lognumber = 0
testlog = [test, lognumber, filelog]
mask = Image.open()
labeled_regions, clabels = nd.label(mask)
regioninfos, testlog = getLabeledInfo(originalimage, Henrybitmap,
labeled_regions,
range(1, clabels + 1), False,
regioninfos, testlog)
regionTree = self.getResults(rundata, imgdata, regions)
return {'region': regionTree, 'mask': maskarray}
def getTomoPixelSize(imagedata):
"""
Get tomography tilt series image pixelsize through the same emtarget
"""
if imagedata["emtarget"] is None:
# uploaded images has no emtarget. This will cause the next query looks
# for all images in the database
if imagedata["label"] != "projection":
return apDatabase.getPixelSize(imagedata)
else:
# Should not trust projection pixel size since it can be different
return None
imageq = leginon.leginondata.AcquisitionImageData(emtarget=imagedata["emtarget"])
imageresults = imageq.query(readimages=False)
for tomoimagedata in imageresults:
if tomoimagedata["label"] != "projection":
predq = leginon.leginondata.TomographyPredictionData(image=tomoimagedata)
results = predq.query(readimages=False)
if results:
return results[0]["pixel size"]
def phaseFlipSpider(self, inimgpath, imgdata):
"""
phaseflip whole image using spider
"""
bestctfvalue = self.getBestCtfValue(imgdata, True)
if bestctfvalue is None:
apDisplay.printWarning("No ctf estimation for current image")
self.badprocess = True
return None
spi_imgpath = os.path.join(self.params['rundir'], self.shortname+".spi")
df1 = abs(bestctfvalue['defocus1'])
df2 = abs(bestctfvalue['defocus2'])
defocus = (df1+df2)/2*1.0e6
apix = apDatabase.getPixelSize(imgdata)
voltage = imgdata['scope']['high tension']
imgsize=imgdata['camera']['dimension']['y']
# find cs
cs = self.getCS(bestctfvalue)
# convert image to spider ###proc2d
emancmd="proc2d %s %s spidersingle"%(inimgpath,spi_imgpath)
apEMAN.executeEmanCmd(emancmd, showcmd = True)
apDisplay.printMsg("phaseflipping entire micrograph with defocus "+str(round(defocus,3))+" microns")
# spider defocus is in Angstroms
defocus *= 10000
outimgpath = filters.phaseFlipImage(spi_imgpath, cs, defocus, voltage, imgsize, apix)
# convert image back to mrc ###proc2d
mrcname = os.path.join(self.params['rundir'], self.shortname+".mrc.corrected.mrc")
emancmd="proc2d %s %s"%(outimgpath,mrcname)
apEMAN.executeEmanCmd(emancmd, showcmd = True)
# remove original spider image
os.remove(spi_imgpath)
return mrcname
def getStackPixelSizeFromStackId(stackId, msg=True):
"""
For a given stack id return stack apix
Not tested on defocal pairs
"""
stackdata = getOnlyStackData(stackId, msg=msg)
if stackdata['pixelsize'] is not None:
### Quicker method
stackapix = stackdata['pixelsize']*1e10
if msg is True:
apDisplay.printMsg("Stack "+str(stackId)+" pixel size: "+str(round(stackapix,3)))
return stackapix
apDisplay.printWarning("Getting stack pixel size from leginon DB, not tested on defocal pairs")
stackpart = getOneParticleFromStackId(stackId, msg=msg)
imgapix = apDatabase.getPixelSize(stackpart['particle']['image'])
stackbin = getStackBinningFromStackId(stackId)
stackapix = imgapix*stackbin
if msg is True:
apDisplay.printMsg("Stack "+str(stackId)+" pixel size: "+str(round(stackapix,3)))
return stackapix
def getTomoPixelSize(imagedata):
'''
Get tomography tilt series image pixelsize through the same emtarget
'''
if imagedata['emtarget'] is None:
# uploaded images has no emtarget. This will cause the next query looks
# for all images in the database
if imagedata['label'] != 'projection':
return apDatabase.getPixelSize(imagedata)
else:
# Should not trust projection pixel size since it can be different
return None
imageq = leginon.leginondata.AcquisitionImageData(
emtarget=imagedata['emtarget'])
imageresults = imageq.query(readimages=False)
for tomoimagedata in imageresults:
if tomoimagedata['label'] != 'projection':
predq = leginon.leginondata.TomographyPredictionData(
image=tomoimagedata)
results = predq.query(readimages=False)
if results:
return results[0]['pixel size']
def processAndSaveFFT(self, imgdata, fftpath):
if os.path.isfile(fftpath):
print "FFT file found"
if fftpath in self.freqdict.keys():
print "Freq found"
return False
print "Freq not found"
print "creating FFT file: ", fftpath
### downsize and filter leginon image
if self.params['uncorrected']:
imgarray = imagefilter.correctImage(imgdata, params)
else:
imgarray = imgdata['image']
### calculate power spectra
apix = apDatabase.getPixelSize(imgdata)
fftarray, freq = ctfpower.power(imgarray,
apix,
mask_radius=0.5,
fieldsize=self.params['fieldsize'])
#fftarray = imagefun.power(fftarray, mask_radius=1)
fftarray = ndimage.median_filter(fftarray, 2)
## preform a rotational average and remove peaks
rotfftarray = ctftools.rotationalAverage2D(fftarray)
stdev = rotfftarray.std()
rotplus = rotfftarray + stdev * 4
fftarray = numpy.where(fftarray > rotplus, rotfftarray, fftarray)
### save to jpeg
self.freqdict[fftpath] = freq
mrc.write(fftarray, fftpath)
self.saveFreqFile()
return True
def tiltPhaseFlipParticles(self, imgdata, imgstackfile, partdatas):
apDisplay.printMsg("Applying per-particle CTF")
ctfvalue = ctfdb.getBestTiltCtfValueForImage(imgdata)
if ctfvalue is None:
apDisplay.printError("Failed to get ctf parameters")
apix = apDatabase.getPixelSize(imgdata)
ctfimgstackfile = os.path.join(self.params['rundir'], apDisplay.short(imgdata['filename'])+"-ctf.hed")
ampconst = ctfvalue['amplitude_contrast']
### calculate defocus at given position
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
CX = dimx/2
CY = dimy/2
if ctfvalue['tilt_axis_angle'] is not None:
N1 = -1.0 * math.sin( math.radians(ctfvalue['tilt_axis_angle']) )
N2 = math.cos( math.radians(ctfvalue['tilt_axis_angle']) )
else:
N1 = 0.0
N2 = 1.0
PSIZE = apix
### High tension on CM is given in kv instead of v so do not divide by 1000 in that case
if imgdata['scope']['tem']['name'] == "CM":
voltage = imgdata['scope']['high tension']
else:
voltage = (imgdata['scope']['high tension'])/1000
# find cs
cs = self.getCS(ctfvalue)
imagicdata = apImagicFile.readImagic(imgstackfile, msg=False)
ctfpartstack = []
for i in range(len(partdatas)):
partdata = partdatas[i]
prepartarray = imagicdata['images'][i]
prepartmrc = "rawpart.dwn.mrc"
postpartmrc = "ctfpart.dwn.mrc"
apImage.arrayToMrc(prepartarray, prepartmrc, msg = False)
### calculate ctf based on position
NX = partdata['xcoord']
NY = dimy-partdata['ycoord'] # reverse due to boxer flip
DX = CX - NX
DY = CY - NY
DF = (N1*DX + N2*DY) * PSIZE * math.tan( math.radians(ctfvalue['tilt_angle']) )
### defocus is in Angstroms
DFL1 = abs(ctfvalue['defocus1'])*1.0e10 + DF
DFL2 = abs(ctfvalue['defocus2'])*1.0e10 + DF
DF_final = (DFL1+DFL2)/2.0
### convert defocus to microns
defocus = DF_final*-1.0e-4
### check to make sure defocus is a reasonable value for applyctf
self.checkDefocus(defocus, apDisplay.short(imgdata['filename']))
parmstr = ("parm=%f,200,1,%.3f,0,17.4,9,1.53,%i,%.1f,%f"
%(defocus, ampconst, voltage, cs, apix))
emancmd = ("applyctf %s %s %s setparm flipphase" % (prepartmrc, postpartmrc, parmstr))
apEMAN.executeEmanCmd(emancmd, showcmd = False)
ctfpartarray = apImage.mrcToArray(postpartmrc, msg=False)
ctfpartstack.append(ctfpartarray)
apImagicFile.writeImagic(ctfpartstack, ctfimgstackfile)
return ctfimgstackfile
def processImage(self, imgdata):
self.ctfvalues = {}
bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True)
apix = apDatabase.getPixelSize(imgdata)
if (not (self.params['onepass'] and self.params['zeropass'])):
maskhighpass = False
ace2inputpath = os.path.join(imgdata['session']['image path'],
imgdata['filename'] + ".mrc")
else:
maskhighpass = True
filterimg = apImage.maskHighPassFilter(imgdata['image'], apix, 1,
self.params['zeropass'],
self.params['onepass'])
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
mrc.write(filterimg, ace2inputpath)
# make sure that the image is a square
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
if dimx != dimy:
dims = [dimx, dimy]
dims.sort()
apDisplay.printMsg("resizing image: %ix%i to %ix%i" %
(dimx, dimy, dims[0], dims[0]))
mrcarray = apImage.mrcToArray(ace2inputpath, msg=False)
clippedmrc = apImage.frame_cut(mrcarray, [dims[0], dims[0]])
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
apImage.arrayToMrc(clippedmrc, ace2inputpath, msg=False)
### pad out image to speed up FFT calculations for non-standard image sizes
print "checking prime factor"
if primefactor.isGoodStack(dimx) is False:
goodsize = primefactor.getNextEvenPrime(dimx)
factor = float(goodsize) / float(dimx)
apDisplay.printMsg("padding image: %ix%i to %ix%i" %
(dimx, dimy, dimx * factor, dimy * factor))
mrcarray = apImage.mrcToArray(ace2inputpath, msg=False)
# paddedmrc = imagefun.pad(mrcarray, None, factor)
paddedmrc = apImage.frame_constant(mrcarray,
(dimx * factor, dimy * factor),
cval=mrcarray.mean())
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
apImage.arrayToMrc(paddedmrc, ace2inputpath, msg=False)
inputparams = {
'input': ace2inputpath,
'cs': self.params['cs'],
'kv': imgdata['scope']['high tension'] / 1000.0,
'apix': apix,
'binby': self.params['bin'],
}
### make standard input for ACE 2
apDisplay.printMsg("Ace2 executable: " + self.ace2exe)
commandline = (self.ace2exe + " -i " + str(inputparams['input']) +
" -b " + str(inputparams['binby']) + " -c " +
str(inputparams['cs']) + " -k " +
str(inputparams['kv']) + " -a " +
str(inputparams['apix']) + " -e " +
str(self.params['edge_b']) + "," +
str(self.params['edge_t']) + " -r " +
str(self.params['rotblur']) + "\n")
### run ace2
apDisplay.printMsg("running ace2 at " + time.asctime())
apDisplay.printColor(commandline, "purple")
t0 = time.time()
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline,
shell=True,
stderr=aceerrf,
stdout=aceoutf)
ace2proc.wait()
### check if ace2 worked
basename = os.path.basename(ace2inputpath)
imagelog = basename + ".ctf.txt"
if not os.path.isfile(imagelog) and self.stats['count'] <= 1:
### ace2 always crashes on first image??? .fft_wisdom file??
time.sleep(1)
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline,
shell=True,
stderr=aceerrf,
stdout=aceoutf)
ace2proc.wait()
if self.params['verbose'] is False:
aceoutf.close()
aceerrf.close()
if not os.path.isfile(imagelog):
lddcmd = "ldd " + self.ace2exe
lddproc = subprocess.Popen(lddcmd, shell=True)
lddproc.wait()
apDisplay.printError("ace2 did not run")
apDisplay.printMsg("ace2 completed in " +
apDisplay.timeString(time.time() - t0))
### parse log file
self.ctfvalues = {
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
}
logf = open(imagelog, "r")
apDisplay.printMsg("reading log file %s" % (imagelog))
for line in logf:
sline = line.strip()
if re.search("^Final Defocus: ", sline):
### old ACE2
apDisplay.printError(
"This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now"
)
#parts = sline.split()
#self.ctfvalues['defocus1'] = float(parts[2])
#self.ctfvalues['defocus2'] = float(parts[3])
### convert to degrees
#self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4]))
elif re.search("^Final Defocus \(m,m,deg\):", sline):
### new ACE2
apDisplay.printMsg("Reading new ACE2 defocus")
parts = sline.split()
#print parts
self.ctfvalues['defocus1'] = float(parts[3])
self.ctfvalues['defocus2'] = float(parts[4])
# ace2 defines negative angle from +x toward +y
self.ctfvalues['angle_astigmatism'] = -float(parts[5])
elif re.search("^Amplitude Contrast:", sline):
parts = sline.split()
self.ctfvalues['amplitude_contrast'] = float(parts[2])
elif re.search("^Confidence:", sline):
parts = sline.split()
self.ctfvalues['confidence'] = float(parts[1])
self.ctfvalues['confidence_d'] = float(parts[1])
logf.close()
### summary stats
apDisplay.printMsg("============")
avgdf = (self.ctfvalues['defocus1'] + self.ctfvalues['defocus2']) / 2.0
ampconst = 100.0 * self.ctfvalues['amplitude_contrast']
pererror = 100.0 * (self.ctfvalues['defocus1'] -
self.ctfvalues['defocus2']) / avgdf
apDisplay.printMsg(
"Defocus: %.3f x %.3f um (%.2f percent astigmatism)" %
(self.ctfvalues['defocus1'] * 1.0e6,
self.ctfvalues['defocus2'] * 1.0e6, pererror))
apDisplay.printMsg("Angle astigmatism: %.2f degrees" %
(self.ctfvalues['angle_astigmatism']))
apDisplay.printMsg("Amplitude contrast: %.2f percent" % (ampconst))
apDisplay.printColor(
"Final confidence: %.3f" % (self.ctfvalues['confidence']), 'cyan')
### double check that the values are reasonable
if avgdf > self.params['maxdefocus'] or avgdf < self.params[
'mindefocus']:
apDisplay.printWarning(
"bad defocus estimate, not committing values to database")
self.badprocess = True
if ampconst < 0.0 or ampconst > 80.0:
apDisplay.printWarning(
"bad amplitude contrast, not committing values to database")
self.badprocess = True
if self.ctfvalues['confidence'] < 0.2:
apDisplay.printWarning(
"bad confidence value, not committing values to database")
self.badprocess = True
## create power spectra jpeg
mrcfile = imgdata['filename'] + ".mrc.edge.mrc"
if os.path.isfile(mrcfile):
jpegfile = os.path.join(
self.powerspecdir,
apDisplay.short(imgdata['filename']) + ".jpg")
ps = apImage.mrcToArray(mrcfile, msg=False)
c = numpy.array(ps.shape) / 2.0
ps[c[0] - 0, c[1] - 0] = ps.mean()
ps[c[0] - 1, c[1] - 0] = ps.mean()
ps[c[0] - 0, c[1] - 1] = ps.mean()
ps[c[0] - 1, c[1] - 1] = ps.mean()
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
ps = numpy.log(ps + 1.0)
ps = (ps - ps.mean()) / ps.std()
cutoff = -2.0 * ps.min()
ps = numpy.where(ps > cutoff, cutoff, ps)
cutoff = ps.mean()
ps = numpy.where(ps < cutoff, cutoff, ps)
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
apImage.arrayToJpeg(ps, jpegfile, msg=False)
apFile.removeFile(mrcfile)
self.ctfvalues['graph3'] = jpegfile
otherfiles = glob.glob(imgdata['filename'] + ".*.txt")
### remove extra debugging files
for filename in otherfiles:
if filename[-9:] == ".norm.txt":
continue
elif filename[-8:] == ".ctf.txt":
continue
else:
apFile.removeFile(filename)
if maskhighpass and os.path.isfile(ace2inputpath):
apFile.removeFile(ace2inputpath)
return
def processImage(self, imgdata):
self.ctfvalues = {}
### convert image to spider
spiderimage = apDisplay.short(imgdata['filename'])+".spi"
spider.write(imgdata['image'], spiderimage)
### high tension in kiloVolts
kvolts = imgdata['scope']['high tension']/1000.0
### spherical aberration in millimeters
cs = apInstrument.getCsValueFromSession(self.getSessionData())
### pixel size in Angstroms
apix = apDatabase.getPixelSize(imgdata)
### write image name
inputstr = ""
inputstr += ("\n")
inputstr += ("# image filename in spider format\n")
inputstr += ("image=%s\n"%(spiderimage))
### common parameters that never change
inputstr += ("\n")
inputstr += ("# common parameters that never change\n")
inputstr += ("N_horizontal=%d\n"%(self.params['fieldsize']))
#inputstr += ("particle_horizontal=%d\n"%(128))
inputstr += ("show_optimization=yes\n")
inputstr += ("micrograph_averaging=yes\n")
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents the typically peak at the center of the PSD to interfere with CTF estimation.
The default value is 0.05, but for micrographs with a very fine sampling this may be lowered towards 0.0
"""
#minres = apix/minfreq => minfreq = apix/minres
minfreq = apix/self.params['resmin']
inputstr += ("min_freq=%.3f\n"%(minfreq))
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents high-resolution terms where only noise exists to interfere with CTF estimation.
The default value is 0.35, but it should be increased for micrographs with signals extending beyond this value
"""
#maxres = apix/maxfreq => maxfreq = apix/maxres
maxfreq = apix/self.params['resmax']
inputstr += ("max_freq=%.3f\n"%(maxfreq))
### CTF input parameters from database
inputstr += ("\n")
inputstr += ("# CTF input parameters from database\n")
inputstr += ("voltage=%d\n"%(kvolts))
inputstr += ("spherical_aberration=%.1f\n"%(cs))
inputstr += ("sampling_rate=%.4f\n"%(apix))
### best defocus in negative Angstroms
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata, msg=True)
if ctfvalue is None:
apDisplay.printWarning("Xmipp CTF as implemented in Appion requires an initial CTF estimate to process"
+"\nPlease run another CTF program first and try again on this image")
self.ctfvalues = None
return
nominal = (ctfvalue['defocus1']+ctfvalue['defocus2'])/2.0
inputstr += ("defocusU=%d\n"%(-abs(ctfvalue['defocus1'])*1.0e10))
inputstr += ("defocusV=%d\n"%(-abs(ctfvalue['defocus2'])*1.0e10))
inputstr += ("Q0=%d\n"%(-ctfvalue['amplitude_contrast']))
inputstr += ("\n")
### open and write to input parameter file
paramfile = apDisplay.short(imgdata['filename'])+"-CTF.prm"
f = open(paramfile, "w")
print inputstr
f.write(inputstr)
f.close()
#[min_freq=<f=0.05>]
#[max_freq=<f=0.35>]
xmippcmd = "xmipp_ctf_estimate_from_micrograph -i %s"%(paramfile)
#xmippcmd = "echo %s"%(paramfile)
t0 = time.time()
apDisplay.printMsg("running ctf estimation at "+time.asctime())
proc = subprocess.Popen(xmippcmd, shell=True, stdout=subprocess.PIPE)
#waittime = 2
#while proc.poll() is None:
# sys.stderr.write(".")
# time.sleep(waittime)
(stdout, stderr) = proc.communicate()
#print (stdout, stderr)
apDisplay.printMsg("ctf estimation completed in "+apDisplay.timeString(time.time()-t0))
### read commandline output to get fit score
lines = stdout.split('\n')
lastline = ""
for line in lines[-50:]:
if "--->" in line:
lastline = line
if not "--->" in lastline:
apDisplay.printWarning("Xmipp CTF failed")
self.badprocess = True
return
bits = lastline.split('--->')
confidence = float(bits[1])
score = round(math.sqrt(1-confidence), 5)
apDisplay.printColor("Confidence value is %.5f (score %.5f)"%(confidence, score), "cyan")
f = open("confidence.log", "a")
f.write("Confidence value is %.5f for image %s (score %.3f)\n"
%(confidence, apDisplay.short(imgdata['filename']), score))
f.close()
### delete image in spider format no longer needed
apFile.removeFile(spiderimage)
### read output parameter file
outparamfile = apDisplay.short(imgdata['filename'])+"_Periodogramavg.ctfparam"
f = open(outparamfile, "r")
for line in f:
sline = line.strip()
bits = sline.split('=')
if sline.startswith("defocusU"):
defocus1 = float(bits[1].strip())
elif sline.startswith("defocusV"):
defocus2 = float(bits[1].strip())
elif sline.startswith("azimuthal_angle"):
angle_astigmatism = float(bits[1].strip())
elif sline.startswith("Q0"):
amplitude_contrast = abs(float(bits[1].strip()))
print defocus1, defocus2, angle_astigmatism, amplitude_contrast
#defocusU= -18418.6
#defocusV= -24272.1
#azimuthal_angle= 79.7936
#Q0= -0.346951 #negative of ACE amplitude_contrast
print "AMP CONTRAST: %.4f -- %.4f"%(amplitude_contrast, ctfvalue['amplitude_contrast'])
self.ctfvalues = {
'defocus1': defocus1*1e-10,
'defocus2': defocus2*1e-10,
'angle_astigmatism': angle_astigmatism,
'amplitude_contrast': amplitude_contrast,
'nominal': nominal,
'defocusinit': nominal,
'confidence_d': score,
'cs': self.params['cs'],
'volts': kvolts*1000.0,
}
return
def processImage(self, imgdata):
"""
time ./ctffind3.exe << eof
micrograph.mrc
montage.pow
2.0, 200.0, 0.07, 60000, 7.0, 2 #! CS[mm], HT[kV], AmpCnst, XMAG, DStep[um], PAve
128, 200.0, 8.0, 5000.0, 40000.0, 5000.0 #! Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep
eof
CARD 1: Input file name for image
CARD 2: Output file name to check result
CARD 3: CS[mm], HT[kV], AmpCnst, XMAG, DStep[um],PAve
CARD 4: Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep, dAst[A]
CTFTILT also asks for TiltA[deg], TiltR[deg] at CARD4
The output image file to check the result of the fitting
shows the filtered average power spectrum of the input
image in one half, and the fitted CTF (squared) in the
other half. The two halves should agree very well for a
successful fit.
CS: Spherical aberration coefficient of the objective in mm
HT: Electron beam voltage in kV
AmpCnst: Amount of amplitude contrast (fraction). For ice
images 0.07, for negative stain about 0.15.
XMAG: Magnification of original image
DStep: Pixel size on scanner in microns, or apix*mag/10000
PAve: Pixel averaging (PAve x PAve) for input image
Box: Tile size. The program divides the image into square
tiles and calculates the average power spectrum. Tiles
with a significantly higher or lower variance are
excluded; these are parts of the image which are unlikely
to contain useful information (beam edge, film number
etc). IMPORTANT: Box must have a even pixel dimensions.
ResMin: Low resolution end of data to be fitted.
ResMaX: High resolution end of data to be fitted.
dFMin: Starting defocus value for grid search in Angstrom.
Positive values represent an underfocus. The program
performs a systematic grid search of defocus values
and astigmatism before fitting a CTF to machine
precision.
dFMax: End defocus value for grid search in Angstrom.
FStep: Step width for grid search in Angstrom.
dAst: An additional parameter, dAst, was added to CARD 4 to restrain
the amount of astigmatism in the CTF fit. This makes the
fitting procedure more robust, especially in cases where
the Thon rings are not easily visible.
TiltA: guessed tilt angle
TiltR: angular range for initial coarse search
"""
#get Defocus in Angstroms
self.ctfvalues = {}
if self.params['nominal'] is not None:
nominal = abs(self.params['nominal']*1e4)
else:
nominal = abs(imgdata['scope']['defocus']*-1.0e10)
ctfvalue = ctfdb.getBestCtfByResolution(imgdata)
if ctfvalue is not None:
"""
## CTFFIND V3.5 (7-March-2012) prefers the smaller of the two values for astigmatic images
I found that say you have an image with 1.1um and 1.5um defocus astigmatism. If you give
CTFFIND the average value of 1.3um for the defocus and 0.4um astig (dast) then it will
try to fit 1.3um and 1.8um, so you need to give it the minimum value (1.1um) for it to
fit 1.1um and 1.5um.
"""
bestdef = min(ctfvalue['defocus1'],ctfvalue['defocus2'])*1.0e10
else:
bestdef = nominal
if ctfvalue is not None and self.params['bestdb'] is True:
bestampcontrast = round(ctfvalue['amplitude_contrast'],3)
beststigdiff = round(abs(ctfvalue['defocus1'] - ctfvalue['defocus2'])*1e10,1)
else:
bestampcontrast = self.params['amp'+self.params['medium']]
beststigdiff = self.params['dast']
if ctfvalue is not None and self.params['bestdb'] is True:
### set res max from resolution_80_percent
gmean = (ctfvalue['resolution_80_percent']*ctfvalue['resolution_50_percent']*self.params['resmax'])**(1/3.)
if gmean < self.params['resmin']:
# replace only if valid Issue #3291, #3547
self.params['resmax'] = round(gmean,2)
apDisplay.printColor("Setting resmax to the geometric mean of resolution values", "purple")
# dstep is the physical detector pixel size
dstep = None
if 'camera' in imgdata and imgdata['camera'] and imgdata['camera']['pixel size']:
dstep = imgdata['camera']['pixel size']['x']
if dstep is None:
dstep = apDatabase.getPixelSize(imgdata)*imgdata['scope']['magnification']/10000.0
dstep /=1e6
dstep = float(dstep)
mpixelsize = apDatabase.getPixelSize(imgdata)*1e-10
if self.params['apix_man'] is not None:
mpixelsize = self.params['apix_man']*1e-10
xmag = dstep / mpixelsize
apDisplay.printMsg("Xmag=%d, dstep=%.2e, mpix=%.2e"%(xmag, dstep, mpixelsize))
inputparams = {
'orig': os.path.join(imgdata['session']['image path'], imgdata['filename']+".mrc"),
'input': apDisplay.short(imgdata['filename'])+".mrc",
'output': apDisplay.short(imgdata['filename'])+"-pow.mrc",
'cs': self.params['cs'],
'kv': imgdata['scope']['high tension']/1000.0,
'ampcnst': bestampcontrast,
'xmag': xmag,
'dstep': dstep*1e6,
'pixavg': self.params['bin'],
'box': self.params['fieldsize'],
'resmin': self.params['resmin'],
'resmax': self.params['resmax'],
'defstep': self.params['defstep'], #round(defocus/32.0, 1),
'dast': beststigdiff,
}
defrange = self.params['defstep'] * self.params['numstep'] ## do 25 steps in either direction
inputparams['defmin']= round(bestdef-defrange, 1) #in meters
if inputparams['defmin'] < 0:
apDisplay.printWarning("Defocus minimum is less than zero")
inputparams['defmin'] = inputparams['defstep']
inputparams['defmax']= round(bestdef+defrange, 1) #in meters
apDisplay.printColor("Defocus search range: %d A to %d A (%.2f to %.2f um)"
%(inputparams['defmin'], inputparams['defmax'],
inputparams['defmin']*1e-4, inputparams['defmax']*1e-4), "cyan")
### secondary lock check right before it starts on the real part
if self.params['parallel'] and os.path.isfile(apDisplay.short(imgdata['filename'])+".mrc"):
# This is a secondary image lock check, checking the first output of the process.
# It alone is not good enough
apDisplay.printWarning('Some other parallel process is working on the same image. Skipping')
return
### create local link to image
if not os.path.exists(inputparams['input']):
os.symlink(inputparams['orig'], inputparams['input'])
### make standard input for ctf estimation
line1cmd = inputparams['input']+"\n"
line2cmd = inputparams['output']+"\n"
line3cmd = (
str(inputparams['cs'])+","
+ str(inputparams['kv'])+","
+ str(inputparams['ampcnst'])+","
+ str(inputparams['xmag'])+","
+ str(inputparams['dstep'])+","
+ str(inputparams['pixavg'])+"\n")
line4cmd = (
str(inputparams['box'])+","
+ str(inputparams['resmin'])+","
+ str(inputparams['resmax'])+","
+ str(inputparams['defmin'])+","
+ str(inputparams['defmax'])+","
+ str(inputparams['defstep'])+","
+ str(inputparams['dast']))
### additional ctftilt parameters
if self.params['ctftilt'] is True:
tiltang = apDatabase.getTiltAngleDeg(imgdata)
line4cmd += (","+str(tiltang)+",10")
line4cmd += "\n"
if os.path.isfile(inputparams['output']):
# program crashes if this file exists
apFile.removeFile(inputparams['output'])
t0 = time.time()
apDisplay.printMsg("running ctf estimation at "+time.asctime())
ctfproglog = os.path.join(self.logdir, os.path.splitext(imgdata['filename'])[0]+"-ctfprog.log")
logf = open(ctfproglog, "w")
ctfprogproc = subprocess.Popen(self.ctfprgmexe, shell=True, stdin=subprocess.PIPE, stdout=logf)
apDisplay.printColor(self.ctfprgmexe, "magenta")
apDisplay.printColor(line1cmd.strip(),"magenta")
apDisplay.printColor(line2cmd.strip(),"magenta")
apDisplay.printColor(line3cmd.strip(),"magenta")
apDisplay.printColor(line4cmd.strip(),"magenta")
ctfprogproc.stdin.write(line1cmd)
ctfprogproc.stdin.write(line2cmd)
ctfprogproc.stdin.write(line3cmd)
ctfprogproc.stdin.write(line4cmd)
ctfprogproc.communicate()
logf.close()
apDisplay.printMsg("ctf estimation completed in "+apDisplay.timeString(time.time()-t0))
#apFile.removeFile(inputparams['input'])
### parse ctf estimation output
self.ctfvalues = {}
logf = open(ctfproglog, "r")
## ctffind & ctftilt have diff # values
numvals = 6
if self.params['ctftilt'] is True:
numvals=8
for line in logf:
sline = line.strip()
if sline[-12:] == "Final Values":
#print sline
if '**********' in sline:
sline = re.sub('**********', ' **********', sline)
bits = sline.split()
if len(bits) != numvals:
apDisplay.printError("wrong number of values in "+str(bits))
for i,bit in enumerate(bits[0:(numvals-2)]):
bits[i] = float(bit)
self.ctfvalues = {
'defocus1': float(bits[0])*1e-10,
'defocus2': float(bits[1])*1e-10,
# WARNING: this is the negative of the direct result
'angle_astigmatism': float(bits[2]),
'amplitude_contrast': inputparams['ampcnst'],
'cross_correlation': float(bits[numvals-3]),
'nominal': nominal*1e-10,
'defocusinit': bestdef*1e-10,
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
'confidence': float(bits[numvals-3]),
'confidence_d': round(math.sqrt(abs(float(bits[numvals-3]))), 5)
}
if self.params['ctftilt'] is True:
self.ctfvalues['tilt_axis_angle']=float(bits[3])
self.ctfvalues['tilt_angle']=float(bits[4])
### write to log file
f = open("ctfvalues.log", "a")
f.write("=== "+imgdata['filename']+" ===\n")
if not self.ctfvalues:
nominaldf = imgdata['scope']['defocus']
else:
nominaldf = self.ctfvalues['nominal']
line1 = ("nominal=%.1e, bestdef=%.1e," %
( nominaldf, self.ctfvalues['defocusinit']))
if self.params['ctftilt'] is True:
self.ctfvalues['origtiltang'] = tiltang
line1+=" tilt=%.1f,"%tiltang
apDisplay.printMsg(line1)
f.write(line1)
line2 = ("def_1=%.1e, def_2=%.1e, astig_angle=%.1f, cross_corr=%.3f,\n" %
( self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'],
self.ctfvalues['cross_correlation'] ))
if self.params['ctftilt'] is True:
line2+= ("tilt_angle=%.1f, tilt_axis_angle=%.1f,\n" %
(self.ctfvalues['tilt_angle'], self.ctfvalues['tilt_axis_angle']))
apDisplay.printMsg(line2)
f.write(line2)
f.close()
#convert powerspectra to JPEG
outputjpgbase = os.path.basename(os.path.splitext(inputparams['output'])[0]+".jpg")
self.lastjpg = outputjpgbase
outputjpg = os.path.join(self.powerspecdir, self.lastjpg)
powspec = apImage.mrcToArray(inputparams['output'])
apImage.arrayToJpeg(powspec, outputjpg)
shutil.move(inputparams['output'], os.path.join(self.powerspecdir, inputparams['output']))
self.ctfvalues['graph1'] = outputjpg
#apFile.removeFile(inputparams['input'])
return
def phaseFlipAceTwo(self, inimgpath, imgdata):
apix = apDatabase.getPixelSize(imgdata)
bestctfvalue = self.getBestCtfValue(imgdata, True)
if bestctfvalue is None:
apDisplay.printWarning("No ctf estimation for current image")
self.badprocess = True
return None
if bestctfvalue['acerun'] is None:
apDisplay.printWarning("No ctf runid for current image")
self.badprocess = True
return None
if bestctfvalue['ctfvalues_file'] is None:
# Since method = ace2 requires a ctfvalues_file,
# create file from database values
### cannot use ACE2 correction without CS value in database
if not 'cs' in bestctfvalue:
apDisplay.printMsg('No spherical abberation value in database, skipping image')
self.badprocess = True
return None
# create ctfvalues_file from ctf run
ctfvaluesfile = "tmp_ctfvaluesfile.txt"
if abs(bestctfvalue['defocus1']) < abs(bestctfvalue['defocus2']):
## this is the canonical form
df1 = abs(bestctfvalue['defocus1'])
df2 = abs(bestctfvalue['defocus2'])
angast = bestctfvalue['angle_astigmatism']
else:
apDisplay.printWarning("|def1| > |def2|, flipping defocus axes")
df1 = abs(bestctfvalue['defocus2'])
df2 = abs(bestctfvalue['defocus1'])
angast = bestctfvalue['angle_astigmatism'] + 90
amp = bestctfvalue['amplitude_contrast']
kv = imgdata['scope']['high tension']/1000
cs = self.getCS(bestctfvalue)/1000
conf = ctfdb.calculateConfidenceScore(bestctfvalue)
if os.path.isfile(ctfvaluesfile):
os.remove(ctfvaluesfile)
f = open(ctfvaluesfile,'w')
f.write("\tFinal Params for image: %s.mrc\n"%imgdata['filename'])
# acecorrect definition is opposite to database
f.write("\tFinal Defocus (m,m,deg): %.6e %.6e %.6f\n"%(df1,df2,-angast))
f.write("\tAmplitude Contrast: %.6f\n"%amp)
f.write("\tVoltage (kV): %.6f\n"%kv)
f.write("\tSpherical Aberration (mm): %.6e\n"%cs)
f.write("\tAngstroms per pixel: %.6e\n"%apix)
f.write("\tConfidence: %.6e\n"%conf)
f.close()
# use ace2 ctfvalues file
else:
ctfvaluesfile = os.path.join(bestctfvalue['acerun']['path']['path'], bestctfvalue['ctfvalues_file'])
ctfvaluesfilesplit = os.path.splitext(ctfvaluesfile)
while ctfvaluesfilesplit[1] != '.mrc':
ctfvaluesfilesplit = os.path.splitext(ctfvaluesfilesplit[0])
ctfvaluesfile = ctfvaluesfilesplit[0]+".mrc.ctf.txt"
defocus1 = bestctfvalue['defocus1']
defocus2 = bestctfvalue['defocus2']
defocus = (defocus1+defocus2)/2.0*1.0e6
apDisplay.printMsg("using ctfvaluesfile: "+ctfvaluesfile)
if not os.path.isfile(ctfvaluesfile):
apDisplay.printError("ctfvaluesfile does not exist")
ace2exe = self.getACE2Path()
outfile = os.path.join(os.getcwd(),imgdata['filename']+".mrc.corrected.mrc")
ace2cmd = (ace2exe+" -ctf %s -apix %.3f -img %s -out %s" % (ctfvaluesfile, apix, inimgpath,outfile))
if self.params['fliptype'] == "ace2image":
ace2cmd += " -wiener 0.1"
apDisplay.printMsg("ace2 command: "+ace2cmd)
apDisplay.printMsg("phaseflipping entire micrograph with defocus "+str(round(defocus,3))+" microns")
#apEMAN.executeEmanCmd(ace2cmd, showcmd = True)
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(ace2cmd, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(ace2cmd, shell=True, stderr=aceerrf, stdout=aceoutf)
ace2proc.wait()
if self.stats['count'] <= 1:
### ace2 always crashes on first image??? .fft_wisdom file??
time.sleep(0.1)
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(ace2cmd, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(ace2cmd, shell=True, stderr=aceerrf, stdout=aceoutf)
ace2proc.wait()
if self.params['verbose'] is False:
aceoutf.close()
aceerrf.close()
if not os.path.isfile(outfile):
apDisplay.printError("ACE 2 failed to create image file:\n%s" % outfile)
return outfile
def _preliminary(self,imgdata):
### set the pixel size
self.params['apix'] = apDatabase.getPixelSize(imgdata)
if not self.params['background']:
apDisplay.printMsg("Pixel size: "+str(self.params['apix']))
def CTFpowerspec(self, imgdata, ctfdata, fftpath=None, fftfreq=None, twod=True):
"""
Make a nice looking powerspectra with lines for location of Thon rings
inputs:
imgdata - sinedon AcquistionImage table row
ctfdata - sinedon apCtfData table row
amplitude constrast - ( a cos + sqrt(1-a^2) sin format)
defocus1 > defocus2
angle - in degrees, positive x-axis is zero
outerbound is now set by self.outerAngstrom1D (in Angstroms)
outside this radius is trimmed away
"""
outerbound = self.outerAngstrom1D * 1e-10
### setup initial parameters for image
self.imgname = imgdata['filename']
if self.debug is True:
print apDisplay.short(self.imgname)
self.powerspecfile = apDisplay.short(self.imgname)+"-powerspec.jpg"
### get peak of CTF
self.cs = ctfdata['cs']*1e-3
self.volts = imgdata['scope']['high tension']
self.ampcontrast = ctfdata['amplitude_contrast']
### process power spectra
self.apix = apDatabase.getPixelSize(imgdata)
if self.debug is True:
print "Pixelsize (A/pix)", self.apix
apDisplay.printMsg("Reading image...")
image = imgdata['image']
self.initfreq = 1./(self.apix * image.shape[0])
self.origimageshape = image.shape
### get correct data
self.convertDefociToConvention(ctfdata)
if self.debug is True:
for key in ctfdata.keys():
if ctfdata[key] is not None and not isinstance(ctfdata[key], dict):
print " ", key, "--", ctfdata[key]
if fftpath is not None and fftfreq is not None and os.path.isfile(fftpath):
powerspec = mrc.read(fftpath).astype(numpy.float64)
self.trimfreq = fftfreq
else:
powerspec, self.trimfreq = ctftools.powerSpectraToOuterResolution(image,
self.outerAngstrom1D, self.apix)
self.trimapix = 1.0/(self.trimfreq * powerspec.shape[0])
#print "Median filter image..."
#powerspec = ndimage.median_filter(powerspec, 2)
apDisplay.printMsg("Preform a rotational average and remove spikes...")
rotfftarray = ctftools.rotationalAverage2D(powerspec)
stdev = rotfftarray.std()
rotplus = rotfftarray + stdev*4
powerspec = numpy.where(powerspec > rotplus, rotfftarray, powerspec)
#print "Light Gaussian blur image..."
#powerspec = ndimage.gaussian_filter(powerspec, 3)
if self.debug is True:
print "\torig pixel %.3f freq %.3e"%(self.apix, self.initfreq)
print "\ttrim pixel %.3f freq %.3e"%(self.trimapix, self.trimfreq)
### more processing
normpowerspec = self.normalizeCtf(powerspec, twod=twod)
if normpowerspec is None:
return None
if twod is True:
self.drawPowerSpecImage(normpowerspec)
ctfdisplaydict = {
'powerspecfile': self.powerspecfile,
'plotsfile': self.plotsfile,
'conf3010': self.conf3010,
'conf5peak': self.conf5peak,
'overconf3010': self.overconf3010,
'overconf5peak': self.overconf5peak,
'res80': self.res80,
'res50': self.res50,
'overres80': self.overres80,
'overres50': self.overres50,
}
return ctfdisplaydict
def run(self):
"""
processes all images
"""
if not self.params['parallel']:
self.cleanParallelLock()
### get images from database
self._getAllImages()
os.chdir(self.params['rundir'])
self.preLoopFunctions()
### start the loop
self.notdone = True
self.badprocess = False
self.stats['startloop'] = time.time()
while self.notdone:
apDisplay.printColor("\nBeginning Main Loop", "green")
imgnum = 0
while imgnum < len(self.imgtree) and self.notdone is True:
self.stats['startimage'] = time.time()
imgdata = self.imgtree[imgnum]
imgnum += 1
### CHECK IF IT IS OKAY TO START PROCESSING IMAGE
if not self._startLoop(imgdata):
continue
### set the pixel size
self.params['apix'] = apDatabase.getPixelSize(imgdata)
if not self.params['background']:
apDisplay.printMsg("Pixel size: " +
str(self.params['apix']))
### START any custom functions HERE:
results = self.loopProcessImage(imgdata)
### WRITE db data
if self.badprocess is False:
if self.params['commit'] is True:
if not self.params['background']:
apDisplay.printColor(
" ==== Committing data to database ==== ",
"blue")
self.loopCommitToDatabase(imgdata)
self.commitResultsToDatabase(imgdata, results)
else:
apDisplay.printWarning(
"not committing results to database, all data will be lost"
)
apDisplay.printMsg(
"to preserve data start script over and add 'commit' flag"
)
self.writeResultsToFiles(imgdata, results)
self.loopCleanUp(imgdata)
else:
apDisplay.printWarning(
"IMAGE FAILED; nothing inserted into database")
self.badprocess = False
self.stats['lastpeaks'] = 0
### FINISH with custom functions
self._writeDoneDict(imgdata['filename'])
if self.params['parallel']:
self.unlockParallel(imgdata.dbid)
loadavg = os.getloadavg()[0]
if loadavg > 2.0:
apDisplay.printMsg("Load average is high " +
str(round(loadavg, 2)))
loadsquared = loadavg * loadavg
time.sleep(loadavg)
apDisplay.printMsg("New load average " +
str(round(os.getloadavg()[0], 2)))
self._printSummary()
if self.params['limit'] is not None and self.stats[
'count'] > self.params['limit']:
apDisplay.printWarning("reached image limit of " +
str(self.params['limit']) +
"; now stopping")
#END LOOP OVER IMAGES
if self.notdone is True:
self.notdone = self._waitForMoreImages()
#END NOTDONE LOOP
self.postLoopFunctions()
self.close()
def uploadTomo(params):
if not params['commit']:
apDisplay.printWarning("not commiting tomogram to database")
return
apDisplay.printMsg("Commiting tomogram to database")
sessiondata = apDatabase.getSessionDataFromSessionName(
params['sessionname'])
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(
params['tiltseriesnumber'], sessiondata)
runname = params['runname']
name = params['name']
if params['full']:
fullbin = params['bin']
else:
fullbin = 1
subbin = params['bin']
alignrun = insertTomoAlignmentRun(sessiondata, None, None, None, None,
fullbin, runname, params['aligndir'],
'manual alignment from upload')
# only tilt series in one alignrun for now
insertTiltsInAlignRun(alignrun, tiltdata, None, True)
alignerdata = insertAlignerParams(alignrun, params)
firstimagedata = getFirstImage(tiltdata)
path = os.path.abspath(params['rundir'])
description = params['description']
if params['full']:
thickness = params['shape'][0] * fullbin
uploadfile = params['zprojfile']
projectimagedata = uploadZProjection(runname, firstimagedata,
uploadfile)
fullrundata = insertFullTomoRun(sessiondata, path, runname, 'upload')
return insertFullTomogram(sessiondata, tiltdata, alignerdata,
fullrundata, name, description,
projectimagedata, thickness, None, fullbin,
[])
else:
projectimagedata = None
fulltomopath = params['rundir'].replace('/' + params['volume'], '')
dummyname = 'dummy'
dummydescription = 'fake full tomogram for subtomogram upload'
thickness = params['shape'][0] * subbin
fullrundata = insertFullTomoRun(sessiondata, fulltomopath, runname,
'upload')
fulltomogram = insertFullTomogram(sessiondata, tiltdata, alignerdata,
fullrundata, dummyname,
dummydescription, projectimagedata,
thickness, None, fullbin, [])
apix = apDatabase.getPixelSize(firstimagedata)
tomoq = appiondata.ApTomogramData()
tomoq['session'] = sessiondata
tomoq['tiltseries'] = tiltdata
results = tomoq.query()
index = len(results) + 1
pixelsize = 1e-10 * apix * subbin
runname = params['volume']
shape = map((lambda x: x * subbin), params['shape'])
dimension = {'x': shape[2], 'y': shape[1], 'z': shape[0]}
subtomorundata = insertSubTomoRun(sessiondata, None, None, runname,
params['invert'], subbin)
return insertSubTomogram(fulltomogram, subtomorundata, None, 0,
dimension, path, name, index, pixelsize,
description)
def processImage(self, imgdata):
self.ctfvalues = {}
bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True)
apix = apDatabase.getPixelSize(imgdata)
if (not (self.params['onepass'] and self.params['zeropass'])):
maskhighpass = False
ace2inputpath = os.path.join(imgdata['session']['image path'],imgdata['filename']+".mrc")
else:
maskhighpass = True
filterimg = apImage.maskHighPassFilter(imgdata['image'],apix,1,self.params['zeropass'],self.params['onepass'])
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
mrc.write(filterimg,ace2inputpath)
# make sure that the image is a square
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
if dimx != dimy:
dims = [dimx,dimy]
dims.sort()
apDisplay.printMsg("resizing image: %ix%i to %ix%i" % (dimx,dimy,dims[0],dims[0]))
mrcarray = apImage.mrcToArray(ace2inputpath,msg=False)
clippedmrc = apImage.frame_cut(mrcarray,[dims[0],dims[0]])
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
apImage.arrayToMrc(clippedmrc,ace2inputpath,msg=False)
### pad out image to speed up FFT calculations for non-standard image sizes
print "checking prime factor"
if primefactor.isGoodStack(dimx) is False:
goodsize = primefactor.getNextEvenPrime(dimx)
factor = float(goodsize) / float(dimx)
apDisplay.printMsg("padding image: %ix%i to %ix%i" % (dimx,dimy,dimx*factor,dimy*factor))
mrcarray = apImage.mrcToArray(ace2inputpath,msg=False)
# paddedmrc = imagefun.pad(mrcarray, None, factor)
paddedmrc = apImage.frame_constant(mrcarray, (dimx*factor,dimy*factor), cval=mrcarray.mean())
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
apImage.arrayToMrc(paddedmrc,ace2inputpath,msg=False)
inputparams = {
'input': ace2inputpath,
'cs': self.params['cs'],
'kv': imgdata['scope']['high tension']/1000.0,
'apix': apix,
'binby': self.params['bin'],
}
### make standard input for ACE 2
apDisplay.printMsg("Ace2 executable: "+self.ace2exe)
commandline = ( self.ace2exe
+ " -i " + str(inputparams['input'])
+ " -b " + str(inputparams['binby'])
+ " -c " + str(inputparams['cs'])
+ " -k " + str(inputparams['kv'])
+ " -a " + str(inputparams['apix'])
+ " -e " + str(self.params['edge_b'])+","+str(self.params['edge_t'])
+ " -r " + str(self.params['rotblur'])
+ "\n" )
### run ace2
apDisplay.printMsg("running ace2 at "+time.asctime())
apDisplay.printColor(commandline, "purple")
t0 = time.time()
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf)
ace2proc.wait()
### check if ace2 worked
basename = os.path.basename(ace2inputpath)
imagelog = basename+".ctf.txt"
if not os.path.isfile(imagelog) and self.stats['count'] <= 1:
### ace2 always crashes on first image??? .fft_wisdom file??
time.sleep(1)
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf)
ace2proc.wait()
if self.params['verbose'] is False:
aceoutf.close()
aceerrf.close()
if not os.path.isfile(imagelog):
lddcmd = "ldd "+self.ace2exe
lddproc = subprocess.Popen(lddcmd, shell=True)
lddproc.wait()
apDisplay.printError("ace2 did not run")
apDisplay.printMsg("ace2 completed in " + apDisplay.timeString(time.time()-t0))
### parse log file
self.ctfvalues = {
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
}
logf = open(imagelog, "r")
apDisplay.printMsg("reading log file %s"%(imagelog))
for line in logf:
sline = line.strip()
if re.search("^Final Defocus: ", sline):
### old ACE2
apDisplay.printError("This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now")
#parts = sline.split()
#self.ctfvalues['defocus1'] = float(parts[2])
#self.ctfvalues['defocus2'] = float(parts[3])
### convert to degrees
#self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4]))
elif re.search("^Final Defocus \(m,m,deg\):", sline):
### new ACE2
apDisplay.printMsg("Reading new ACE2 defocus")
parts = sline.split()
#print parts
self.ctfvalues['defocus1'] = float(parts[3])
self.ctfvalues['defocus2'] = float(parts[4])
# ace2 defines negative angle from +x toward +y
self.ctfvalues['angle_astigmatism'] = -float(parts[5])
elif re.search("^Amplitude Contrast:",sline):
parts = sline.split()
self.ctfvalues['amplitude_contrast'] = float(parts[2])
elif re.search("^Confidence:",sline):
parts = sline.split()
self.ctfvalues['confidence'] = float(parts[1])
self.ctfvalues['confidence_d'] = float(parts[1])
logf.close()
### summary stats
apDisplay.printMsg("============")
avgdf = (self.ctfvalues['defocus1']+self.ctfvalues['defocus2'])/2.0
ampconst = 100.0*self.ctfvalues['amplitude_contrast']
pererror = 100.0 * (self.ctfvalues['defocus1']-self.ctfvalues['defocus2']) / avgdf
apDisplay.printMsg("Defocus: %.3f x %.3f um (%.2f percent astigmatism)"%
(self.ctfvalues['defocus1']*1.0e6, self.ctfvalues['defocus2']*1.0e6, pererror ))
apDisplay.printMsg("Angle astigmatism: %.2f degrees"%(self.ctfvalues['angle_astigmatism']))
apDisplay.printMsg("Amplitude contrast: %.2f percent"%(ampconst))
apDisplay.printColor("Final confidence: %.3f"%(self.ctfvalues['confidence']),'cyan')
### double check that the values are reasonable
if avgdf > self.params['maxdefocus'] or avgdf < self.params['mindefocus']:
apDisplay.printWarning("bad defocus estimate, not committing values to database")
self.badprocess = True
if ampconst < 0.0 or ampconst > 80.0:
apDisplay.printWarning("bad amplitude contrast, not committing values to database")
self.badprocess = True
if self.ctfvalues['confidence'] < 0.2:
apDisplay.printWarning("bad confidence value, not committing values to database")
self.badprocess = True
## create power spectra jpeg
mrcfile = imgdata['filename']+".mrc.edge.mrc"
if os.path.isfile(mrcfile):
jpegfile = os.path.join(self.powerspecdir, apDisplay.short(imgdata['filename'])+".jpg")
ps = apImage.mrcToArray(mrcfile,msg=False)
c = numpy.array(ps.shape)/2.0
ps[c[0]-0,c[1]-0] = ps.mean()
ps[c[0]-1,c[1]-0] = ps.mean()
ps[c[0]-0,c[1]-1] = ps.mean()
ps[c[0]-1,c[1]-1] = ps.mean()
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
ps = numpy.log(ps+1.0)
ps = (ps-ps.mean())/ps.std()
cutoff = -2.0*ps.min()
ps = numpy.where(ps > cutoff, cutoff, ps)
cutoff = ps.mean()
ps = numpy.where(ps < cutoff, cutoff, ps)
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
apImage.arrayToJpeg(ps, jpegfile, msg=False)
apFile.removeFile(mrcfile)
self.ctfvalues['graph3'] = jpegfile
otherfiles = glob.glob(imgdata['filename']+".*.txt")
### remove extra debugging files
for filename in otherfiles:
if filename[-9:] == ".norm.txt":
continue
elif filename[-8:] == ".ctf.txt":
continue
else:
apFile.removeFile(filename)
if maskhighpass and os.path.isfile(ace2inputpath):
apFile.removeFile(ace2inputpath)
return
def getMicrographPixelSizeFromStackId(stackid, msg=True):
stackPart = getStackParticlesFromId(stackid, msg=True)[0]
pixelsize = apDatabase.getPixelSize(stackPart['particle']['image'])
return pixelsize
def getMicrographPixelSizeFromStackId(stackid, msg=True): stackPart = getStackParticlesFromId(stackid, msg=True)[0] pixelsize = apDatabase.getPixelSize(stackPart['particle']['image']) return pixelsize
def function(self, rundata, imgdata, binnedimgarray):
self.params['apix'] = apDatabase.getPixelSize(imgdata)
regions, maskarray = apCrud.makeMask(self.params, binnedimgarray)
regionTree = self.getResults(rundata, imgdata, regions)
return {'region': regionTree, 'mask': maskarray}
def run(self):
"""
processes all images
"""
if not self.params['parallel']:
self.cleanParallelLock()
### get images from database
self._getAllImages()
os.chdir(self.params['rundir'])
self.preLoopFunctions()
### start the loop
self.notdone=True
self.badprocess = False
self.stats['startloop'] = time.time()
while self.notdone:
apDisplay.printColor("\nBeginning Main Loop", "green")
imgnum = 0
while imgnum < len(self.imgtree) and self.notdone is True:
self.stats['startimage'] = time.time()
imgdata = self.imgtree[imgnum]
imgnum += 1
### CHECK IF IT IS OKAY TO START PROCESSING IMAGE
if not self._startLoop(imgdata):
continue
### set the pixel size
self.params['apix'] = apDatabase.getPixelSize(imgdata)
if not self.params['background']:
apDisplay.printMsg("Pixel size: "+str(self.params['apix']))
### START any custom functions HERE:
results = self.loopProcessImage(imgdata)
### WRITE db data
if self.badprocess is False:
if self.params['commit'] is True:
if not self.params['background']:
apDisplay.printColor(" ==== Committing data to database ==== ", "blue")
self.loopCommitToDatabase(imgdata)
self.commitResultsToDatabase(imgdata, results)
else:
apDisplay.printWarning("not committing results to database, all data will be lost")
apDisplay.printMsg("to preserve data start script over and add 'commit' flag")
self.writeResultsToFiles(imgdata, results)
self.loopCleanUp(imgdata)
else:
apDisplay.printWarning("IMAGE FAILED; nothing inserted into database")
self.badprocess = False
self.stats['lastpeaks'] = 0
### FINISH with custom functions
self._writeDoneDict(imgdata['filename'])
if self.params['parallel']:
self.unlockParallel(imgdata.dbid)
loadavg = os.getloadavg()[0]
if loadavg > 2.0:
apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
loadsquared = loadavg*loadavg
time.sleep(loadavg)
apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2)))
self._printSummary()
if self.params['limit'] is not None and self.stats['count'] > self.params['limit']:
apDisplay.printWarning("reached image limit of "+str(self.params['limit'])+"; now stopping")
#END LOOP OVER IMAGES
if self.notdone is True:
self.notdone = self._waitForMoreImages()
#END NOTDONE LOOP
self.postLoopFunctions()
self.close()
def processImage(self, imgdata):
"""
time ./ctffind3.exe << eof
Input image file name [input.mrc] : 15aug13neil2_14jul14d_05sq_012hl_02ed-a.mrc
Output diagnostic filename
[diagnostic_output.mrc] : 15aug13neil2_14jul14d_05sq_012hl_02ed-a-pow.mrc
Pixel size [1.0] : 2.7
Acceleration voltage [300.0] : 300
Spherical aberration [2.7] : 2.7
Amplitude contrast [0.07] : 0.07
Size of power spectrum to compute [512] : 512
Minimum resolution [30.0] : 20
Maximum resolution [5.0] : 5
Minimum defocus [5000.0] :
Maximum defocus [50000.0] :
Defocus search step [500.0] :
Expected (tolerated) astigmatism [100.0] :
Find additional phase shift? [no] :
"""
paramInputOrder = ['input', 'output', 'apix', 'kv', 'cs', 'ampcontrast', 'fieldsize',
'resmin', 'resmax', 'defmin', 'defmax', 'defstep', 'expect_astig', 'phase', 'newline',]
#get Defocus in Angstroms
self.ctfvalues = {}
nominal = abs(imgdata['scope']['defocus']*-1.0e10)
ctfvalue = ctfdb.getBestCtfByResolution(imgdata)
if ctfvalue is not None:
"""
## CTFFIND V3.5 (7-March-2012) prefers the smaller of the two values for astigmatic images
I found that say you have an image with 1.1um and 1.5um defocus astigmatism. If you give
CTFFIND the average value of 1.3um for the defocus and 0.4um astig (dast) then it will
try to fit 1.3um and 1.8um, so you need to give it the minimum value (1.1um) for it to
fit 1.1um and 1.5um.
"""
bestdef = min(ctfvalue['defocus1'],ctfvalue['defocus2'])*1.0e10
else:
bestdef = nominal
if ctfvalue is not None and self.params['bestdb'] is True:
bestampcontrast = round(ctfvalue['amplitude_contrast'],3)
beststigdiff = round(abs(ctfvalue['defocus1'] - ctfvalue['defocus2'])*1e10,1)
else:
bestampcontrast = self.params['ampcontrast']
beststigdiff = self.params['dast']*10000.
if ctfvalue is not None and self.params['bestdb'] is True:
### set res max from resolution_80_percent
gmean = (ctfvalue['resolution_80_percent']*ctfvalue['resolution_50_percent']*self.params['resmax'])**(1/3.)
if gmean < self.params['resmin']*0.9:
# replace only if valid Issue #3291
self.params['resmax'] = round(gmean,2)
apDisplay.printColor("Setting resmax to the geometric mean of resolution values", "purple")
# dstep is the physical detector pixel size
apix = apDatabase.getPixelSize(imgdata)
# inputparams defocii and astig are in Angstroms
inputparams = {
'orig': os.path.join(imgdata['session']['image path'], imgdata['filename']+".mrc"),
'input': apDisplay.short(imgdata['filename'])+".mrc",
'output': apDisplay.short(imgdata['filename'])+"-pow.mrc",
'apix': apix,
'kv': imgdata['scope']['high tension']/1000.0,
'cs': self.params['cs'],
'ampcontrast': bestampcontrast,
'fieldsize': self.params['fieldsize'],
'resmin': self.params['resmin'],
'resmax': self.params['resmax'],
'defstep': self.params['defstep']*10000., #round(defocus/32.0, 1),
'expect_astig': beststigdiff,
'phase': 'no', # this is a secondary amp contrast term for phase plates
'newline': '\n',
}
defrange = self.params['defstep'] * self.params['numstep'] * 1e4 ## do 25 steps in either direction # in angstrum
inputparams['defmin']= round(bestdef-defrange, 1) #in angstrom
if inputparams['defmin'] < 0:
apDisplay.printWarning("Defocus minimum is less than zero")
inputparams['defmin'] = inputparams['defstep']
inputparams['defmax']= round(bestdef+defrange, 1) #in angstrom
apDisplay.printColor("Defocus search range: %d A to %d A (%.2f to %.2f um)"
%(inputparams['defmin'], inputparams['defmax'],
inputparams['defmin']*1e-4, inputparams['defmax']*1e-4), "cyan")
### secondary lock check right before it starts on the real part
if self.params['parallel'] and os.path.isfile(apDisplay.short(imgdata['filename'])+".mrc"):
# This is a secondary image lock check, checking the first output of the process.
# It alone is not good enough
apDisplay.printWarning('Some other parallel process is working on the same image. Skipping')
return
### create local link to image
if not os.path.exists(inputparams['input']):
os.symlink(inputparams['orig'], inputparams['input'])
if os.path.isfile(inputparams['output']):
# program crashes if this file exists
apFile.removeFile(inputparams['output'])
t0 = time.time()
apDisplay.printMsg("running ctf estimation at "+time.asctime())
for paramName in paramInputOrder:
apDisplay.printColor("%s = %s"%(paramName,inputparams[paramName]),"magenta")
print ""
ctfprogproc = subprocess.Popen(self.ctfprgmexe, shell=True, stdin=subprocess.PIPE,)
apDisplay.printColor(self.ctfprgmexe, "magenta")
for paramName in paramInputOrder:
apDisplay.printColor(inputparams[paramName],"magenta")
ctfprogproc.stdin.write(str(inputparams[paramName])+'\n')
ctfprogproc.communicate()
tdiff = time.time()-t0
apDisplay.printMsg("ctf estimation completed in "+apDisplay.timeString(tdiff))
if tdiff < 1.0:
apDisplay.printError("Failed to run CTFFIND4 program...")
### cannot run ctffind_plot_results.sh on CentOS 6
# This script requires gnuplot version >= 4.6, but you have version 4.2
### parse ctf estimation output
self.ctfvalues = {}
ctfproglog = apDisplay.short(imgdata['filename'])+"-pow.txt"
apDisplay.printMsg("reading %s"%(ctfproglog))
logf = open(ctfproglog, "r")
for line in logf:
sline = line.strip()
if sline.startswith('#'):
continue
bits = sline.split()
self.ctfvalues = {
'imagenum': int(float(bits[0])),
'defocus2': float(bits[1])*1e-10,
'defocus1': float(bits[2])*1e-10,
'angle_astigmatism': float(bits[3]),
'extra_phase': float(bits[4]),
'amplitude_contrast': inputparams['ampcontrast'],
'cross_correlation': float(bits[5]),
'ctffind4_resolution': float(bits[6]),
'defocusinit': bestdef*1e-10,
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
'confidence': float(bits[5]),
'confidence_d': round(math.sqrt(abs(float(bits[5]))), 5)
}
print self.ctfvalues
#convert powerspectra to JPEG
outputjpgbase = apDisplay.short(imgdata['filename'])+"-pow.jpg"
self.lastjpg = outputjpgbase
outputjpg = os.path.join(self.powerspecdir, self.lastjpg)
powspec = apImage.mrcToArray(inputparams['output'])
apImage.arrayToJpeg(powspec, outputjpg)
shutil.move(inputparams['output'], os.path.join(self.powerspecdir, inputparams['output']))
self.ctfvalues['graph1'] = outputjpg
#apFile.removeFile(inputparams['input'])
return
def processImage(self, imgdata):
self.ctfvalues = {}
### convert image to spider
spiderimage = apDisplay.short(imgdata['filename']) + ".spi"
spider.write(imgdata['image'], spiderimage)
### high tension in kiloVolts
kvolts = imgdata['scope']['high tension'] / 1000.0
### spherical aberration in millimeters
cs = apInstrument.getCsValueFromSession(self.getSessionData())
### pixel size in Angstroms
apix = apDatabase.getPixelSize(imgdata)
### write image name
inputstr = ""
inputstr += ("\n")
inputstr += ("# image filename in spider format\n")
inputstr += ("image=%s\n" % (spiderimage))
### common parameters that never change
inputstr += ("\n")
inputstr += ("# common parameters that never change\n")
inputstr += ("N_horizontal=%d\n" % (self.params['fieldsize']))
#inputstr += ("particle_horizontal=%d\n"%(128))
inputstr += ("show_optimization=yes\n")
inputstr += ("micrograph_averaging=yes\n")
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents the typically peak at the center of the PSD to interfere with CTF estimation.
The default value is 0.05, but for micrographs with a very fine sampling this may be lowered towards 0.0
"""
#minres = apix/minfreq => minfreq = apix/minres
minfreq = apix / self.params['resmin']
inputstr += ("min_freq=%.3f\n" % (minfreq))
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents high-resolution terms where only noise exists to interfere with CTF estimation.
The default value is 0.35, but it should be increased for micrographs with signals extending beyond this value
"""
#maxres = apix/maxfreq => maxfreq = apix/maxres
maxfreq = apix / self.params['resmax']
inputstr += ("max_freq=%.3f\n" % (maxfreq))
### CTF input parameters from database
inputstr += ("\n")
inputstr += ("# CTF input parameters from database\n")
inputstr += ("voltage=%d\n" % (kvolts))
inputstr += ("spherical_aberration=%.1f\n" % (cs))
inputstr += ("sampling_rate=%.4f\n" % (apix))
### best defocus in negative Angstroms
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata, msg=True)
nominal = (ctfvalue['defocus1'] + ctfvalue['defocus2']) / 2.0
inputstr += ("defocusU=%d\n" % (-abs(ctfvalue['defocus1']) * 1.0e10))
inputstr += ("defocusV=%d\n" % (-abs(ctfvalue['defocus2']) * 1.0e10))
inputstr += ("Q0=%d\n" % (-ctfvalue['amplitude_contrast']))
inputstr += ("\n")
### open and write to input parameter file
paramfile = apDisplay.short(imgdata['filename']) + "-CTF.prm"
f = open(paramfile, "w")
print inputstr
f.write(inputstr)
f.close()
#[min_freq=<f=0.05>]
#[max_freq=<f=0.35>]
xmippcmd = "xmipp_ctf_estimate_from_micrograph -i %s" % (paramfile)
#xmippcmd = "echo %s"%(paramfile)
t0 = time.time()
apDisplay.printMsg("running ctf estimation at " + time.asctime())
proc = subprocess.Popen(xmippcmd, shell=True, stdout=subprocess.PIPE)
#waittime = 2
#while proc.poll() is None:
# sys.stderr.write(".")
# time.sleep(waittime)
(stdout, stderr) = proc.communicate()
#print (stdout, stderr)
apDisplay.printMsg("ctf estimation completed in " +
apDisplay.timeString(time.time() - t0))
### read commandline output to get fit score
lines = stdout.split('\n')
lastline = ""
for line in lines[-50:]:
if "--->" in line:
lastline = line
if not "--->" in lastline:
apDisplay.printWarning("Xmipp CTF failed")
self.badprocess = True
return
bits = lastline.split('--->')
confidence = float(bits[1])
score = round(math.sqrt(1 - confidence), 5)
apDisplay.printColor(
"Confidence value is %.5f (score %.5f)" % (confidence, score),
"cyan")
f = open("confidence.log", "a")
f.write("Confidence value is %.5f for image %s (score %.3f)\n" %
(confidence, apDisplay.short(imgdata['filename']), score))
f.close()
### delete image in spider format no longer needed
apFile.removeFile(spiderimage)
### read output parameter file
outparamfile = apDisplay.short(
imgdata['filename']) + "_Periodogramavg.ctfparam"
f = open(outparamfile, "r")
for line in f:
sline = line.strip()
bits = sline.split('=')
if sline.startswith("defocusU"):
defocus1 = float(bits[1].strip())
elif sline.startswith("defocusV"):
defocus2 = float(bits[1].strip())
elif sline.startswith("azimuthal_angle"):
angle_astigmatism = float(bits[1].strip())
elif sline.startswith("Q0"):
amplitude_contrast = abs(float(bits[1].strip()))
print defocus1, defocus2, angle_astigmatism, amplitude_contrast
#defocusU= -18418.6
#defocusV= -24272.1
#azimuthal_angle= 79.7936
#Q0= -0.346951 #negative of ACE amplitude_contrast
print "AMP CONTRAST: %.4f -- %.4f" % (amplitude_contrast,
ctfvalue['amplitude_contrast'])
self.ctfvalues = {
'defocus1': defocus1 * 1e-10,
'defocus2': defocus2 * 1e-10,
'angle_astigmatism': angle_astigmatism,
'amplitude_contrast': amplitude_contrast,
'nominal': nominal,
'defocusinit': nominal,
'confidence_d': score,
'cs': self.params['cs'],
'volts': kvolts * 1000.0,
}
return
def _preliminary(self, imgdata):
### set the pixel size
self.params['apix'] = apDatabase.getPixelSize(imgdata)
if not self.params['background']:
apDisplay.printMsg("Pixel size: " + str(self.params['apix']))
def function(self,rundata, imgdata, binnedimgarray):
self.params['apix'] = apDatabase.getPixelSize(imgdata)
regions,maskarray = apCrud.makeMask(self.params, binnedimgarray)
regionTree = self.getResults(rundata, imgdata, regions)
return {'region':regionTree,'mask':maskarray}
def processImage(self, imgdata):
"""
time ./ctffind3.exe << eof
micrograph.mrc
montage.pow
2.0, 200.0, 0.07, 60000, 7.0, 2 #! CS[mm], HT[kV], AmpCnst, XMAG, DStep[um], PAve
128, 200.0, 8.0, 5000.0, 40000.0, 5000.0 #! Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep
eof
CARD 1: Input file name for image
CARD 2: Output file name to check result
CARD 3: CS[mm], HT[kV], AmpCnst, XMAG, DStep[um],PAve
CARD 4: Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep, dAst[A]
CTFTILT also asks for TiltA[deg], TiltR[deg] at CARD4
The output image file to check the result of the fitting
shows the filtered average power spectrum of the input
image in one half, and the fitted CTF (squared) in the
other half. The two halves should agree very well for a
successful fit.
CS: Spherical aberration coefficient of the objective in mm
HT: Electron beam voltage in kV
AmpCnst: Amount of amplitude contrast (fraction). For ice
images 0.07, for negative stain about 0.15.
XMAG: Magnification of original image
DStep: Pixel size on scanner in microns, or apix*mag/10000
PAve: Pixel averaging (PAve x PAve) for input image
Box: Tile size. The program divides the image into square
tiles and calculates the average power spectrum. Tiles
with a significantly higher or lower variance are
excluded; these are parts of the image which are unlikely
to contain useful information (beam edge, film number
etc). IMPORTANT: Box must have a even pixel dimensions.
ResMin: Low resolution end of data to be fitted.
ResMaX: High resolution end of data to be fitted.
dFMin: Starting defocus value for grid search in Angstrom.
Positive values represent an underfocus. The program
performs a systematic grid search of defocus values
and astigmatism before fitting a CTF to machine
precision.
dFMax: End defocus value for grid search in Angstrom.
FStep: Step width for grid search in Angstrom.
dAst: An additional parameter, dAst, was added to CARD 4 to restrain
the amount of astigmatism in the CTF fit. This makes the
fitting procedure more robust, especially in cases where
the Thon rings are not easily visible.
TiltA: guessed tilt angle
TiltR: angular range for initial coarse search
"""
#get Defocus in Angstroms
self.ctfvalues = {}
nominal = abs(imgdata['scope']['defocus'] * -1.0e10)
ctfvalue = ctfdb.getBestCtfByResolution(imgdata)
if ctfvalue is not None:
bestdef = abs(ctfvalue['defocus1'] +
ctfvalue['defocus2']) / 2.0 * 1.0e10
else:
bestdef = nominal
if ctfvalue is not None and self.params['bestdb'] is True:
bestampcontrast = ctfvalue['amplitude_contrast']
beststigdiff = abs(ctfvalue['defocus1'] -
ctfvalue['defocus2']) * 1e10
else:
bestampcontrast = self.params['amp' + self.params['medium']]
beststigdiff = self.params['dast']
# dstep is the physical detector pixel size
dstep = None
if 'camera' in imgdata and imgdata['camera'] and imgdata['camera'][
'pixel size']:
dstep = imgdata['camera']['pixel size']['x']
if dstep is None:
dstep = apDatabase.getPixelSize(
imgdata) * imgdata['scope']['magnification'] / 10000.0
dstep /= 1e6
dstep = float(dstep)
mpixelsize = apDatabase.getPixelSize(imgdata) * 1e-10
xmag = dstep / mpixelsize
apDisplay.printMsg("Xmag=%d, dstep=%.2e, mpix=%.2e" %
(xmag, dstep, mpixelsize))
inputparams = {
'orig':
os.path.join(imgdata['session']['image path'],
imgdata['filename'] + ".mrc"),
'input':
apDisplay.short(imgdata['filename']) + ".mrc",
'output':
apDisplay.short(imgdata['filename']) + "-pow.mrc",
'cs':
self.params['cs'],
'kv':
imgdata['scope']['high tension'] / 1000.0,
'ampcnst':
bestampcontrast,
'xmag':
xmag,
'dstep':
dstep * 1e6,
'pixavg':
self.params['bin'],
'box':
self.params['fieldsize'],
'resmin':
self.params['resmin'],
'resmax':
self.params['resmax'],
'defstep':
self.params['defstep'], #round(defocus/32.0, 1),
'dast':
beststigdiff,
}
defrange = self.params['defstep'] * self.params[
'numstep'] ## do 25 steps in either direction
inputparams['defmin'] = round(bestdef - defrange, 1) #in meters
if inputparams['defmin'] < 0:
apDisplay.printWarning("Defocus minimum is less than zero")
inputparams['defmin'] = inputparams['defstep']
inputparams['defmax'] = round(bestdef + defrange, 1) #in meters
apDisplay.printColor(
"Defocus search range: %d A to %d A (%.2f to %.2f um)" %
(inputparams['defmin'], inputparams['defmax'],
inputparams['defmin'] * 1e-4, inputparams['defmax'] * 1e-4),
"cyan")
### create local link to image
if not os.path.exists(inputparams['input']):
cmd = "ln -s " + inputparams['orig'] + " " + inputparams[
'input'] + "\n"
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
### make standard input for ctf estimation
line1cmd = inputparams['input'] + "\n"
line2cmd = inputparams['output'] + "\n"
line3cmd = (str(inputparams['cs']) + "," + str(inputparams['kv']) +
"," + str(inputparams['ampcnst']) + "," +
str(inputparams['xmag']) + "," +
str(inputparams['dstep']) + "," +
str(inputparams['pixavg']) + "\n")
line4cmd = (str(inputparams['box']) + "," +
str(inputparams['resmin']) + "," +
str(inputparams['resmax']) + "," +
str(inputparams['defmin']) + "," +
str(inputparams['defmax']) + "," +
str(inputparams['defstep']) + "," +
str(inputparams['dast']))
### additional ctftilt parameters
if self.params['ctftilt'] is True:
tiltang = apDatabase.getTiltAngleDeg(imgdata)
line4cmd += ("," + str(tiltang) + ",10")
line4cmd += "\n"
if os.path.isfile(inputparams['output']):
# program crashes if this file exists
apFile.removeFile(inputparams['output'])
t0 = time.time()
apDisplay.printMsg("running ctf estimation at " + time.asctime())
ctfproglog = os.path.join(
self.logdir,
os.path.splitext(imgdata['filename'])[0] + "-ctfprog.log")
logf = open(ctfproglog, "w")
ctfprogproc = subprocess.Popen(self.ctfprgmexe,
shell=True,
stdin=subprocess.PIPE,
stdout=logf)
apDisplay.printColor(self.ctfprgmexe, "magenta")
apDisplay.printColor(line1cmd.strip(), "magenta")
apDisplay.printColor(line2cmd.strip(), "magenta")
apDisplay.printColor(line3cmd.strip(), "magenta")
apDisplay.printColor(line4cmd.strip(), "magenta")
ctfprogproc.stdin.write(line1cmd)
ctfprogproc.stdin.write(line2cmd)
ctfprogproc.stdin.write(line3cmd)
ctfprogproc.stdin.write(line4cmd)
ctfprogproc.communicate()
logf.close()
apDisplay.printMsg("ctf estimation completed in " +
apDisplay.timeString(time.time() - t0))
#apFile.removeFile(inputparams['input'])
### parse ctf estimation output
self.ctfvalues = {}
logf = open(ctfproglog, "r")
## ctffind & ctftilt have diff # values
numvals = 6
if self.params['ctftilt'] is True:
numvals = 8
for line in logf:
sline = line.strip()
if sline[-12:] == "Final Values":
#print sline
if '**********' in sline:
sline = re.sub('**********', ' **********', sline)
bits = sline.split()
if len(bits) != numvals:
apDisplay.printError("wrong number of values in " +
str(bits))
for i, bit in enumerate(bits[0:(numvals - 2)]):
bits[i] = float(bit)
self.ctfvalues = {
'defocus1':
float(bits[0]) * 1e-10,
'defocus2':
float(bits[1]) * 1e-10,
# WARNING: this is the negative of the direct result
'angle_astigmatism':
float(bits[2]),
'amplitude_contrast':
inputparams['ampcnst'],
'cross_correlation':
float(bits[numvals - 3]),
'nominal':
nominal * 1e-10,
'defocusinit':
bestdef * 1e-10,
'cs':
self.params['cs'],
'volts':
imgdata['scope']['high tension'],
'confidence':
float(bits[numvals - 3]),
'confidence_d':
round(math.sqrt(abs(float(bits[numvals - 3]))), 5)
}
if self.params['ctftilt'] is True:
self.ctfvalues['tilt_axis_angle'] = float(bits[3])
self.ctfvalues['tilt_angle'] = float(bits[4])
### write to log file
f = open("ctfvalues.log", "a")
f.write("=== " + imgdata['filename'] + " ===\n")
line1 = ("nominal=%.1e, bestdef=%.1e," %
(self.ctfvalues['nominal'], self.ctfvalues['defocusinit']))
if self.params['ctftilt'] is True:
self.ctfvalues['origtiltang'] = tiltang
line1 += " tilt=%.1f," % tiltang
apDisplay.printMsg(line1)
f.write(line1)
line2 = (
"def_1=%.1e, def_2=%.1e, astig_angle=%.1f, cross_corr=%.3f,\n" %
(self.ctfvalues['defocus1'], self.ctfvalues['defocus2'],
self.ctfvalues['angle_astigmatism'],
self.ctfvalues['cross_correlation']))
if self.params['ctftilt'] is True:
line2 += ("tilt_angle=%.1f, tilt_axis_angle=%.1f,\n" %
(self.ctfvalues['tilt_angle'],
self.ctfvalues['tilt_axis_angle']))
apDisplay.printMsg(line2)
f.write(line2)
f.close()
#convert powerspectra to JPEG
outputjpgbase = os.path.basename(
os.path.splitext(inputparams['output'])[0] + ".jpg")
self.lastjpg = outputjpgbase
outputjpg = os.path.join(self.powerspecdir, self.lastjpg)
powspec = apImage.mrcToArray(inputparams['output'])
apImage.arrayToJpeg(powspec, outputjpg)
shutil.move(inputparams['output'],
os.path.join(self.powerspecdir, inputparams['output']))
self.ctfvalues['graph1'] = outputjpg
#apFile.removeFile(inputparams['input'])
return
class MakeAlignedSumLoop(appionPBS.AppionPBS):
#=====================
def setupParserOptions(self):
configuration_options = deProcessFrames.ConfigurationOptions( )
options_list = configuration_options.get_options_list( )
sections = options_list.keys( )
for section in sections :
for option in options_list[ section ] :
if section == 'gainreference' or section == 'darkreference':
if option['name'] in ['filename', 'framecount']:
continue
if section == 'boxes':
if option['name'] in ['fromlist','fromonefile','fromfiles','boxsize','minimum']:
continue
if section == 'input' and option['name']=='framecount':
continue
if section == 'radiationdamage':
# if option['name'] in ['apix', 'voltage']:
if option['name'] in ['voltage']:
continue
# if section == 'alignment' and option['name']=='correct':
# continue
if option[ 'type' ] == str :
metavar = "STR"
elif option[ 'type' ] == int :
metavar = "INT"
elif option[ 'type' ] == float :
metavar = "FLOAT"
self.parser.add_option( "--%s_%s" % ( section, option[ 'name' ] ), type = option[ 'type' ], metavar = metavar, help = option[ 'help' ], default=option['default'] )
self.parser.add_option("--alignlabel", dest="alignlabel", default='a',
help="label to be appended to the presetname, e.g. --label=a gives ed-a as the aligned preset for preset ed", metavar="CHAR")
self.parser.add_option("--border", dest='border', type='int', default=0, help='Clip border specified border pixels and pad back out with mean value')
self.parser.add_option("--hackcopy", dest='hackcopy', action='store_true', default=False, help='Copy corrected image to session directory and overwrite the original image, saving the orignal with a new extension ".orig.mrc"')
#=======================
def checkConflicts(self):
#if override-dark or bright selected, should check for override-darkframes
pass
def getFrameType(self):
# set how frames are saved depending on what is found in the basepath
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
if sessiondata['frame path']:
# 3.0+
return ddinfo.getRawFrameType(sessiondata['frame path'])
else:
# pre-3.0
return ddinfo.getRawFrameType(sessiondata['image path'])
#=======================
def preLoopFunctions(self):
self.dd = apDDprocess.initializeDDFrameprocess(self.params['sessionname'],self.params['wait'])
self.dd.setRunDir(self.params['rundir'])
self.dd.setRawFrameType(self.getFrameType())
self.dd.setDoseFDriftCorrOptions(self.params)
self.exposurerate_is_default = self.params['radiationdamage_exposurerate'] == 1.0
self.imageids = []
# Optimize AppionLoop wait time for this since the processing now takes longer than
# image acquisition
self.setWaitSleepMin(0.4)
self.setProcessBatchCount(1)
self.params['output_fileformat'] = 'mrc'
def getTargets(self, imgdata, scratchdir='', handlefiles='direct'):
targetdict={}
#copy flatfields
brightrefpath=imgdata['bright']['session']['image path']
brightrefname=imgdata['bright']['filename']+'.mrc'
brightref=os.path.join(brightrefpath,brightrefname)
darkrefpath=imgdata['dark']['session']['image path']
darkrefname=imgdata['dark']['filename']+'.mrc'
darkref=os.path.join(darkrefpath,darkrefname)
#################################### do away with override flatfield option
framesdirname=imgdata['filename']+'.frames'
apDisplay.printMsg('Copying frames %s' % (framesdirname))
framespath=imgdata['session']['frame path']
framespathname=os.path.join(framespath,framesdirname)
if handlefiles == 'direct':
targetdict['brightref']=brightref
targetdict['darkref']=darkref
targetdict['framespathname']=framespathname
targetdict['outpath']=self.params['rundir']
elif handlefiles == 'copy':
shutil.copy(brightref,scratchdir)
shutil.copy(darkref,scratchdir)
targetdict['brightref']=os.path.join(scratchdir,brightrefname)
targetdict['darkref']=os.path.join(scratchdir,darkrefname)
try:
shutil.copytree(framespathname,os.path.join(scratchdir, framesdirname))
except:
apDisplay.printWarning('there was a problem copying the frames for %s' % (imgdata['filename']))
targetdict['framespathname']=os.path.join(scratchdir,framesdirname)
targetdict['outpath']=os.path.join(scratchdir,imgdata['filename'])
elif handlefiles == 'link':
os.symlink(brightref,os.path.join(scratchdir,brightrefname))
os.symlink(darkref,os.path.join(scratchdir,darkrefname))
os.symlink(framespathname,os.path.join(scratchdir, framesdirname))
targetdict['brightref']=os.path.join(scratchdir,brightrefname)
targetdict['darkref']=os.path.join(scratchdir,darkrefname)
targetdict['framespathname']=os.path.join(scratchdir,framesdirname)
targetdict['outpath']=os.path.join(scratchdir,imgdata['filename'])
return targetdict
def calculateListDifference(self,list1,list2):
from sets import Set
set1 = Set(list1)
set2 = Set(list2)
list_diff = list(set1.difference(set2))
list_diff.sort()
return list_diff
def getCameraDefects(self, imgdata):
"""
Set defects for camera in self.params if not entered already.
"""
corrector_plan = imgdata['corrector plan']
cam_size = imgdata['camera']['dimension']
border = self.params['border']
# map name to de params name and leginon corrector plan name
namemap = {'x':('columns','cols'),'y':('rows','rows')}
if not corrector_plan:
return
for axis in namemap.keys():
de_name = 'defects_%s' % namemap[axis][0]
leg_name = 'bad_%s' % namemap[axis][1]
exclude_list = []
# figure out the defects if not specified already
if not self.params[de_name] and corrector_plan[leg_name]:
# Do not include a location in defects for DE
# process if in the border because large number
# of defect correction is slow.
if border:
exclude_list = range(0,border)
exclude_list.extend(range(cam_size[axis]-border,cam_size[axis]))
bad = self.calculateListDifference(corrector_plan[leg_name],exclude_list)
self.params[de_name] = ','.join(map((lambda x: '%d' % x),bad))
# TODO: need to handle bad pixels, too
def generateCommand(self, imgdata, targetdict):
# need to avoid non-frame saved image for proper caching
if imgdata is None or imgdata['camera']['save frames'] != True:
apDisplay.printWarning('%s skipped for no-frame-saved\n ' % imgdata['filename'])
return
### set processing image
try:
self.dd.setImageData(imgdata)
except Exception, e:
apDisplay.printWarning(e.args[0])
return
# Align
# Doing the alignment
kev=imgdata['scope']['high tension']/1000
apix=apDatabase.getPixelSize(imgdata)
nframes=imgdata['camera']['nframes']
try:
dose=apDatabase.getDoseFromImageData(imgdata)
except:
dose=None
# overwrite radiationdamage_exposurerate if it is at default
if dose and self.exposurerate_is_default:
self.params['radiationdamage_exposurerate']=dose/nframes
#set appion specific options
#flatfield references
self.params['gainreference_filename']=targetdict['brightref']
brightnframes=imgdata['bright']['camera']['nframes']
self.params['gainreference_framecount']=brightnframes
self.params['darkreference_filename']=targetdict['darkref']
darknframes=imgdata['dark']['camera']['nframes']
self.params['darkreference_framecount']=darknframes
self.getCameraDefects(imgdata)
self.params['input_framecount']=nframes
#self.params['run_verbosity']=3
self.params['output_invert']=0
self.params['radiationdamage_apix']=apix
self.params['radiationdamage_voltage']=kev
#self.params['boxes_boxsize']=boxsize
if os.path.exists(targetdict['outpath']):
shutil.rmtree(targetdict['outpath'])
os.mkdir(targetdict['outpath'])
command=['runDEProcessFrames.py']
keys=self.params.keys()
keys.sort()
for key in keys:
param=self.params[key]
#print key, param, type(param)
if param == None or param=='' or key=='description':
pass
else:
option='--%s=%s' % (key,param)
command.append(option)
command.append(targetdict['outpath'])
#framespath=imgdata['session']['frame path']
#framespathname=os.path.join(framespath,imgdata['filename']+'.frames')
framespathname=targetdict['framespathname']
#check to see if there are frames in the path
framesinpath=len(glob.glob(os.path.join(framespathname,'*')))
if framesinpath == 0:
apDisplay.printWarning('%s skipped because %d frames were found' % (imgdata['filename'],framesinpath))
return
command.append(framespathname)
return command
