def trimPowerSpectraToOuterResolution(powerspec, outerresolution, freq):
"""
freq and outerresolution must have same units (e.g., Anstroms or meters)
resolution = (# columns) * apix / (pixel distance from center)
therefore:
pixel distance from center = (# columns) * apix / resolution
"""
if debug is True:
print "trimPowerSpectraToOuterResolution()"
imagewidth = powerspec.shape[0]
initmaxres = 2.0 / (freq * imagewidth)
if debug is True:
print "__Image shape %d x %d" % (powerspec.shape[0],
powerspec.shape[1])
print "__Frequeny %.3e" % (freq)
print "__Resolution request %.3f" % (outerresolution)
print "__Init max resolution %.3f" % (initmaxres)
if initmaxres > outerresolution:
apDisplay.printWarning(
"Requested resolution (%.3f) is not available (%.3f)" %
(outerresolution, initmaxres))
outerresolution = initmaxres
pixellimitradius = min(powerspec.shape[0], powerspec.shape[1]) / 2
else:
pixellimitradius = int(math.ceil(1. / (freq * outerresolution)))
goodpixellimitradius = primefactor.getNextEvenPrime(pixellimitradius)
# If outerresolution input is larger than initmaxres, the way goodpixellimitradius
# is calculated above may make the result larger than half the image dimension.
while goodpixellimitradius > min(powerspec.shape[0],
powerspec.shape[1]) / 2:
pixellimitradius -= 2
if debug is True:
print "__Recalculate pixel limit dimension with: ", pixellimitradius
goodpixellimitradius = primefactor.getNextEvenPrime(pixellimitradius)
finalres = 1. / (freq * goodpixellimitradius)
if debug is True:
print "__Pixel limit dimension: ", goodpixellimitradius
print "__Final max resolution %.3f" % (finalres)
### convert to diameter and trim
newshape = (goodpixellimitradius * 2, goodpixellimitradius * 2)
if debug is True:
print "__Trimming image"
trimpowerspec = imagefilter.frame_cut(powerspec, newshape)
if newshape != trimpowerspec.shape:
apDisplay.printError(
"shape mismatch for frame_cut (%d,%d) --> (%d,%d) = (%d,%d)" %
(powerspec.shape[0], powerspec.shape[1], newshape[0], newshape[1],
trimpowerspec.shape[0], trimpowerspec.shape[1]))
if debug is True:
print "original image size %d x %d" % (powerspec.shape)
print "trimmed image size %d x %d" % (trimpowerspec.shape)
return trimpowerspec
def trimPowerSpectraToOuterResolution(powerspec, outerresolution, freq):
"""
freq and outerresolution must have same units (e.g., Anstroms or meters)
resolution = (# columns) * apix / (pixel distance from center)
therefore:
pixel distance from center = (# columns) * apix / resolution
"""
if debug is True:
print "trimPowerSpectraToOuterResolution()"
imagewidth = powerspec.shape[0]
initmaxres = 2.0/(freq*imagewidth)
if debug is True:
print "__Image shape %d x %d"%(powerspec.shape[0], powerspec.shape[1])
print "__Frequeny %.3e"%(freq)
print "__Resolution request %.3f"%(outerresolution)
print "__Init max resolution %.3f"%(initmaxres)
if initmaxres > outerresolution:
apDisplay.printWarning("Requested resolution (%.3f) is not available (%.3f)"
%(outerresolution, initmaxres))
outerresolution = initmaxres
pixellimitradius = min(powerspec.shape[0],powerspec.shape[1]) / 2
else:
pixellimitradius = int(math.ceil(1./(freq * outerresolution)))
goodpixellimitradius = primefactor.getNextEvenPrime(pixellimitradius)
# If outerresolution input is larger than initmaxres, the way goodpixellimitradius
# is calculated above may make the result larger than half the image dimension.
while goodpixellimitradius > min(powerspec.shape[0],powerspec.shape[1])/2:
pixellimitradius -= 2
if debug is True:
print "__Recalculate pixel limit dimension with: ", pixellimitradius
goodpixellimitradius = primefactor.getNextEvenPrime(pixellimitradius)
finalres = 1./(freq * goodpixellimitradius)
if debug is True:
print "__Pixel limit dimension: ", goodpixellimitradius
print "__Final max resolution %.3f"%(finalres)
### convert to diameter and trim
newshape = (goodpixellimitradius*2, goodpixellimitradius*2)
if debug is True:
print "__Trimming image"
trimpowerspec = imagefilter.frame_cut(powerspec, newshape)
if newshape != trimpowerspec.shape:
apDisplay.printError("shape mismatch for frame_cut (%d,%d) --> (%d,%d) = (%d,%d)"
%(powerspec.shape[0],powerspec.shape[1],
newshape[0],newshape[1],
trimpowerspec.shape[0],trimpowerspec.shape[1]))
if debug is True:
print "original image size %d x %d"%(powerspec.shape)
print "trimmed image size %d x %d"%(trimpowerspec.shape)
return trimpowerspec
def runRefineCTF(self, imgdata, fftpath):
### reset important values
self.bestres = 1e10
self.bestellipse = None
self.bestvalues = {}
self.volts = imgdata['scope']['high tension']
self.wavelength = ctftools.getTEMLambda(self.volts)
## get value in meters
self.cs = apInstrument.getCsValueFromSession(
self.getSessionData()) * 1e-3
self.ctfvalues = {
'volts': self.volts,
'wavelength': self.wavelength,
'cs': self.cs,
}
### need to get FFT file open and freq of said file
fftarray = mrc.read(fftpath).astype(numpy.float64)
self.freq = self.freqdict[fftpath]
### convert resolution limit into pixel distance
fftwidth = fftarray.shape[0]
maxres = 2.0 / (self.freq * fftwidth)
if maxres > self.params['reslimit']:
apDisplay.printWarning(
"Cannot get requested res %.1fA higher than max Nyquist resolution %.1fA"
% (self.params['reslimit'], maxres))
self.params['reslimit'] = math.ceil(maxres)
limitwidth = int(math.ceil(2.0 /
(self.params['reslimit'] * self.freq)))
limitwidth = primefactor.getNextEvenPrime(limitwidth)
requestres = 2.0 / (self.freq * limitwidth)
if limitwidth > fftwidth:
apDisplay.printError("Cannot get requested resolution" + (
" request res %.1fA higher than max res %.1fA for new widths %d > %d"
% (requestres, maxres, limitwidth, fftwidth)))
apDisplay.printColor(
"Requested resolution OK: " +
(" request res %.1fA less than max res %.1fA with fft widths %d < %d"
% (requestres, maxres, limitwidth, fftwidth)), "green")
newshape = (limitwidth, limitwidth)
fftarray = imagefilter.frame_cut(fftarray, newshape)
### spacing parameters
self.mfreq = self.freq * 1e10
fftwidth = min(fftarray.shape)
self.apix = 1.0 / (fftwidth * self.freq)
self.ctfvalues['apix'] = self.apix
### print message
bestDbValues = ctfdb.getBestCtfByResolution(imgdata)
if bestDbValues is None:
apDisplay.printColor(
"SKIPPING: No CTF values for image %s" %
(apDisplay.short(imgdata['filename'])), "red")
self.badprocess = True
return
### skip if resolution > 90.
if bestDbValues['resolution_50_percent'] > 90.:
apDisplay.printColor(
"SKIPPING: No decent CTF values for image %s" %
(apDisplay.short(imgdata['filename'])), "yellow")
self.badprocess = True
return
self.ctfvalues['amplitude_contrast'] = bestDbValues[
'amplitude_contrast']
lowerrad1 = ctftools.getCtfExtrema(
bestDbValues['defocus1'], self.mfreq, self.cs, self.volts,
self.ctfvalues['amplitude_contrast'], 1, "valley")
lowerrad2 = ctftools.getCtfExtrema(
bestDbValues['defocus2'], self.mfreq, self.cs, self.volts,
self.ctfvalues['amplitude_contrast'], 1, "valley")
meanRad = (lowerrad1[0] + lowerrad2[0]) / 2.0
self.ellipseParams = {
'a': lowerrad1[0],
'b': lowerrad2[0],
'alpha': math.radians(bestDbValues['angle_astigmatism']),
}
ellipratio = self.ellipseParams['a'] / self.ellipseParams['b']
defratio = bestDbValues['defocus2'] / bestDbValues['defocus1']
print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f" % (ellipratio, defratio,
math.sqrt(defratio))
self.bestvalues['defocus'] = (bestDbValues['defocus1'] +
bestDbValues['defocus2']) / 2.0
raddata, PSDarray = self.from2Dinto1D(fftarray)
lowerbound = numpy.searchsorted(raddata, meanRad * self.freq)
upperbound = numpy.searchsorted(raddata, 1 / 10.)
###
# This is the start of the actual program
###
self.refineEllipseLoop(fftarray, lowerbound, upperbound)
##==================================
## FINISH UP
##==================================
apDisplay.printColor("Finishing up using best found CTF values",
"blue")
self.printBestValues()
### take best values and use them
self.ctfvalues = self.bestvalues
self.ellipseParams = self.bestellipse
### stupid fix, get value in millimeters
self.ctfvalues['cs'] = apInstrument.getCsValueFromSession(
self.getSessionData())
### translate ellipse into ctf values
if self.ellipseParams is not None:
self.ctfvalues['angle_astigmatism'] = math.degrees(
self.ellipseParams['alpha'])
ellipratio = self.ellipseParams['a'] / self.ellipseParams['b']
phi = math.asin(self.ctfvalues['amplitude_contrast'])
#note: a > b then def1 < def2
#major axis
self.ctfvalues['defocus1'] = self.ctfvalues['defocus'] / ellipratio
#minor axis
self.ctfvalues['defocus2'] = self.ctfvalues['defocus'] * ellipratio
defdiff = 1.0 - 2 * self.ctfvalues['defocus'] / (
self.ctfvalues['defocus1'] + self.ctfvalues['defocus2'])
print "%.3e --> %.3e,%.3e" % (self.ctfvalues['defocus'],
self.ctfvalues['defocus2'],
self.ctfvalues['defocus1'])
print defdiff * 100
if defdiff * 100 > 1:
apDisplay.printWarning("Large astigmatism")
#sys.exit(1)
else:
ellipratio = 1.0
self.ctfvalues['angle_astigmatism'] = 0.0
self.ctfvalues['defocus1'] = self.ctfvalues['defocus']
self.ctfvalues['defocus2'] = self.ctfvalues['defocus']
self.badprocess = True
return
try:
if self.ctfvalues['amplitude_contrast'] < self.minAmpCon:
self.ctfvalues['amplitude_contrast'] = self.minAmpCon
if self.ctfvalues['amplitude_contrast'] > self.maxAmpCon:
self.ctfvalues['amplitude_contrast'] = self.maxAmpCon
except KeyError:
pass
if abs(self.ctfvalues['defocus1']) > abs(self.ctfvalues['defocus2']):
# incorrect, need to shift angle by 90 degrees
apDisplay.printWarning("|def1| > |def2|, flipping defocus axes")
tempdef = self.ctfvalues['defocus1']
self.ctfvalues['defocus1'] = self.ctfvalues['defocus2']
self.ctfvalues['defocus2'] = tempdef
self.ctfvalues['angle_astigmatism'] += 90
# get astig_angle within range -90 < angle <= 90
while self.ctfvalues['angle_astigmatism'] > 90:
self.ctfvalues['angle_astigmatism'] -= 180
while self.ctfvalues['angle_astigmatism'] < -90:
self.ctfvalues['angle_astigmatism'] += 180
avgres = self.getResolution(self.ctfvalues['defocus'], raddata,
PSDarray, lowerbound)
apDisplay.printColor(
"Final defocus values %.3e -> %.3e, %.3e; ac=%.2f, res=%.1f" %
(self.ctfvalues['defocus'], self.ctfvalues['defocus1'],
self.ctfvalues['defocus2'], self.ctfvalues['amplitude_contrast'],
avgres / 2.0), "green")
for i in range(10):
print "===================================="
print "PREVIOUS VALUES"
ctfdb.getBestCtfByResolution(imgdata)
print "CURRENT VALUES"
defocusratio = self.ctfvalues['defocus2'] / self.ctfvalues['defocus1']
apDisplay.printColor(
"def1: %.2e | def2: %.2e | angle: %.1f | ampcontr %.2f | defratio %.3f"
% (self.ctfvalues['defocus1'], self.ctfvalues['defocus2'],
self.ctfvalues['angle_astigmatism'],
self.ctfvalues['amplitude_contrast'], defocusratio), "blue")
self.printBestValues()
#ellipratio = self.ellipseParams['a']/self.ellipseParams['b']
print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f" % (
ellipratio, defocusratio, math.sqrt(defocusratio))
print "===================================="
return
def processImage(self, imgdata):
self.ctfvalues = {}
bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True)
apix = apDatabase.getPixelSize(imgdata)
if (not (self.params['onepass'] and self.params['zeropass'])):
maskhighpass = False
ace2inputpath = os.path.join(imgdata['session']['image path'],imgdata['filename']+".mrc")
else:
maskhighpass = True
filterimg = apImage.maskHighPassFilter(imgdata['image'],apix,1,self.params['zeropass'],self.params['onepass'])
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
mrc.write(filterimg,ace2inputpath)
# make sure that the image is a square
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
if dimx != dimy:
dims = [dimx,dimy]
dims.sort()
apDisplay.printMsg("resizing image: %ix%i to %ix%i" % (dimx,dimy,dims[0],dims[0]))
mrcarray = apImage.mrcToArray(ace2inputpath,msg=False)
clippedmrc = apImage.frame_cut(mrcarray,[dims[0],dims[0]])
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
apImage.arrayToMrc(clippedmrc,ace2inputpath,msg=False)
### pad out image to speed up FFT calculations for non-standard image sizes
print "checking prime factor"
if primefactor.isGoodStack(dimx) is False:
goodsize = primefactor.getNextEvenPrime(dimx)
factor = float(goodsize) / float(dimx)
apDisplay.printMsg("padding image: %ix%i to %ix%i" % (dimx,dimy,dimx*factor,dimy*factor))
mrcarray = apImage.mrcToArray(ace2inputpath,msg=False)
# paddedmrc = imagefun.pad(mrcarray, None, factor)
paddedmrc = apImage.frame_constant(mrcarray, (dimx*factor,dimy*factor), cval=mrcarray.mean())
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
apImage.arrayToMrc(paddedmrc,ace2inputpath,msg=False)
inputparams = {
'input': ace2inputpath,
'cs': self.params['cs'],
'kv': imgdata['scope']['high tension']/1000.0,
'apix': apix,
'binby': self.params['bin'],
}
### make standard input for ACE 2
apDisplay.printMsg("Ace2 executable: "+self.ace2exe)
commandline = ( self.ace2exe
+ " -i " + str(inputparams['input'])
+ " -b " + str(inputparams['binby'])
+ " -c " + str(inputparams['cs'])
+ " -k " + str(inputparams['kv'])
+ " -a " + str(inputparams['apix'])
+ " -e " + str(self.params['edge_b'])+","+str(self.params['edge_t'])
+ " -r " + str(self.params['rotblur'])
+ "\n" )
### run ace2
apDisplay.printMsg("running ace2 at "+time.asctime())
apDisplay.printColor(commandline, "purple")
t0 = time.time()
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf)
ace2proc.wait()
### check if ace2 worked
basename = os.path.basename(ace2inputpath)
imagelog = basename+".ctf.txt"
if not os.path.isfile(imagelog) and self.stats['count'] <= 1:
### ace2 always crashes on first image??? .fft_wisdom file??
time.sleep(1)
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf)
ace2proc.wait()
if self.params['verbose'] is False:
aceoutf.close()
aceerrf.close()
if not os.path.isfile(imagelog):
lddcmd = "ldd "+self.ace2exe
lddproc = subprocess.Popen(lddcmd, shell=True)
lddproc.wait()
apDisplay.printError("ace2 did not run")
apDisplay.printMsg("ace2 completed in " + apDisplay.timeString(time.time()-t0))
### parse log file
self.ctfvalues = {
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
}
logf = open(imagelog, "r")
apDisplay.printMsg("reading log file %s"%(imagelog))
for line in logf:
sline = line.strip()
if re.search("^Final Defocus: ", sline):
### old ACE2
apDisplay.printError("This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now")
#parts = sline.split()
#self.ctfvalues['defocus1'] = float(parts[2])
#self.ctfvalues['defocus2'] = float(parts[3])
### convert to degrees
#self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4]))
elif re.search("^Final Defocus \(m,m,deg\):", sline):
### new ACE2
apDisplay.printMsg("Reading new ACE2 defocus")
parts = sline.split()
#print parts
self.ctfvalues['defocus1'] = float(parts[3])
self.ctfvalues['defocus2'] = float(parts[4])
# ace2 defines negative angle from +x toward +y
self.ctfvalues['angle_astigmatism'] = -float(parts[5])
elif re.search("^Amplitude Contrast:",sline):
parts = sline.split()
self.ctfvalues['amplitude_contrast'] = float(parts[2])
elif re.search("^Confidence:",sline):
parts = sline.split()
self.ctfvalues['confidence'] = float(parts[1])
self.ctfvalues['confidence_d'] = float(parts[1])
logf.close()
### summary stats
apDisplay.printMsg("============")
avgdf = (self.ctfvalues['defocus1']+self.ctfvalues['defocus2'])/2.0
ampconst = 100.0*self.ctfvalues['amplitude_contrast']
pererror = 100.0 * (self.ctfvalues['defocus1']-self.ctfvalues['defocus2']) / avgdf
apDisplay.printMsg("Defocus: %.3f x %.3f um (%.2f percent astigmatism)"%
(self.ctfvalues['defocus1']*1.0e6, self.ctfvalues['defocus2']*1.0e6, pererror ))
apDisplay.printMsg("Angle astigmatism: %.2f degrees"%(self.ctfvalues['angle_astigmatism']))
apDisplay.printMsg("Amplitude contrast: %.2f percent"%(ampconst))
apDisplay.printColor("Final confidence: %.3f"%(self.ctfvalues['confidence']),'cyan')
### double check that the values are reasonable
if avgdf > self.params['maxdefocus'] or avgdf < self.params['mindefocus']:
apDisplay.printWarning("bad defocus estimate, not committing values to database")
self.badprocess = True
if ampconst < 0.0 or ampconst > 80.0:
apDisplay.printWarning("bad amplitude contrast, not committing values to database")
self.badprocess = True
if self.ctfvalues['confidence'] < 0.2:
apDisplay.printWarning("bad confidence value, not committing values to database")
self.badprocess = True
## create power spectra jpeg
mrcfile = imgdata['filename']+".mrc.edge.mrc"
if os.path.isfile(mrcfile):
jpegfile = os.path.join(self.powerspecdir, apDisplay.short(imgdata['filename'])+".jpg")
ps = apImage.mrcToArray(mrcfile,msg=False)
c = numpy.array(ps.shape)/2.0
ps[c[0]-0,c[1]-0] = ps.mean()
ps[c[0]-1,c[1]-0] = ps.mean()
ps[c[0]-0,c[1]-1] = ps.mean()
ps[c[0]-1,c[1]-1] = ps.mean()
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
ps = numpy.log(ps+1.0)
ps = (ps-ps.mean())/ps.std()
cutoff = -2.0*ps.min()
ps = numpy.where(ps > cutoff, cutoff, ps)
cutoff = ps.mean()
ps = numpy.where(ps < cutoff, cutoff, ps)
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
apImage.arrayToJpeg(ps, jpegfile, msg=False)
apFile.removeFile(mrcfile)
self.ctfvalues['graph3'] = jpegfile
otherfiles = glob.glob(imgdata['filename']+".*.txt")
### remove extra debugging files
for filename in otherfiles:
if filename[-9:] == ".norm.txt":
continue
elif filename[-8:] == ".ctf.txt":
continue
else:
apFile.removeFile(filename)
if maskhighpass and os.path.isfile(ace2inputpath):
apFile.removeFile(ace2inputpath)
return
def runRefineCTF(self, imgdata, fftpath):
### reset important values
self.bestres = 1e10
self.bestellipse = None
self.bestvalues = {}
self.volts = imgdata['scope']['high tension']
self.wavelength = ctftools.getTEMLambda(self.volts)
## get value in meters
self.cs = apInstrument.getCsValueFromSession(self.getSessionData())*1e-3
self.ctfvalues = {
'volts': self.volts,
'wavelength': self.wavelength,
'cs': self.cs,
}
### need to get FFT file open and freq of said file
fftarray = mrc.read(fftpath).astype(numpy.float64)
self.freq = self.freqdict[fftpath]
### convert resolution limit into pixel distance
fftwidth = fftarray.shape[0]
maxres = 2.0/(self.freq*fftwidth)
if maxres > self.params['reslimit']:
apDisplay.printError("Cannot get requested res %.1fA higher than max res %.1fA"
%(maxres, self.params['reslimit']))
limitwidth = int(math.ceil(2.0/(self.params['reslimit']*self.freq)))
limitwidth = primefactor.getNextEvenPrime(limitwidth)
requestres = 2.0/(self.freq*limitwidth)
if limitwidth > fftwidth:
apDisplay.printError("Cannot get requested resolution"
+(" request res %.1fA higher than max res %.1fA for new widths %d > %d"
%(requestres, maxres, limitwidth, fftwidth)))
apDisplay.printColor("Requested resolution OK: "
+(" request res %.1fA less than max res %.1fA with fft widths %d < %d"
%(requestres, maxres, limitwidth, fftwidth)), "green")
newshape = (limitwidth, limitwidth)
fftarray = imagefilter.frame_cut(fftarray, newshape)
### spacing parameters
self.mfreq = self.freq*1e10
fftwidth = min(fftarray.shape)
self.apix = 1.0/(fftwidth*self.freq)
self.ctfvalues['apix'] = self.apix
### print message
bestDbValues = ctfdb.getBestCtfByResolution(imgdata)
if bestDbValues is None:
apDisplay.printColor("SKIPPING: No CTF values for image %s"
%(apDisplay.short(imgdata['filename'])), "red")
self.badprocess = True
return
### skip if resolution > 90.
if bestDbValues['resolution_50_percent'] > 90.:
apDisplay.printColor("SKIPPING: No decent CTF values for image %s"
%(apDisplay.short(imgdata['filename'])), "yellow")
self.badprocess = True
return
self.ctfvalues['amplitude_contrast'] = bestDbValues['amplitude_contrast']
lowerrad1 = ctftools.getCtfExtrema(bestDbValues['defocus1'], self.mfreq, self.cs, self.volts,
self.ctfvalues['amplitude_contrast'], 1, "valley")
lowerrad2 = ctftools.getCtfExtrema(bestDbValues['defocus2'], self.mfreq, self.cs, self.volts,
self.ctfvalues['amplitude_contrast'], 1, "valley")
meanRad = (lowerrad1[0] + lowerrad2[0])/2.0
self.ellipseParams = {
'a': lowerrad1[0],
'b': lowerrad2[0],
'alpha': math.radians(bestDbValues['angle_astigmatism']),
}
ellipratio = self.ellipseParams['a']/self.ellipseParams['b']
defratio = bestDbValues['defocus2']/bestDbValues['defocus1']
print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f"%(ellipratio, defratio, math.sqrt(defratio))
self.bestvalues['defocus'] = (bestDbValues['defocus1']+bestDbValues['defocus2'])/2.0
raddata, PSDarray = self.from2Dinto1D(fftarray)
lowerbound = numpy.searchsorted(raddata, meanRad*self.freq)
upperbound = numpy.searchsorted(raddata, 1/10.)
###
# This is the start of the actual program
###
self.refineEllipseLoop(fftarray, lowerbound, upperbound)
##==================================
## FINISH UP
##==================================
apDisplay.printColor("Finishing up using best found CTF values", "blue")
self.printBestValues()
### take best values and use them
self.ctfvalues = self.bestvalues
self.ellipseParams = self.bestellipse
### stupid fix, get value in millimeters
self.ctfvalues['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
### translate ellipse into ctf values
if self.ellipseParams is not None:
self.ctfvalues['angle_astigmatism'] = math.degrees(self.ellipseParams['alpha'])
ellipratio = self.ellipseParams['a']/self.ellipseParams['b']
phi = math.asin(self.ctfvalues['amplitude_contrast'])
#note: a > b then def1 < def2
#major axis
self.ctfvalues['defocus1'] = self.ctfvalues['defocus']/ellipratio
#minor axis
self.ctfvalues['defocus2'] = self.ctfvalues['defocus']*ellipratio
defdiff = 1.0 - 2*self.ctfvalues['defocus']/(self.ctfvalues['defocus1']+self.ctfvalues['defocus2'])
print "%.3e --> %.3e,%.3e"%(self.ctfvalues['defocus'],
self.ctfvalues['defocus2'], self.ctfvalues['defocus1'])
print defdiff*100
if defdiff*100 > 1:
apDisplay.printWarning("Large astigmatism")
#sys.exit(1)
else:
ellipratio = 1.0
self.ctfvalues['angle_astigmatism'] = 0.0
self.ctfvalues['defocus1'] = self.ctfvalues['defocus']
self.ctfvalues['defocus2'] = self.ctfvalues['defocus']
if self.ctfvalues['amplitude_contrast'] < 0.0:
self.ctfvalues['amplitude_contrast'] = 0.0
if self.ctfvalues['amplitude_contrast'] > self.maxAmpCon:
self.ctfvalues['amplitude_contrast'] = self.maxAmpCon
if abs(self.ctfvalues['defocus1']) > abs(self.ctfvalues['defocus2']):
# incorrect, need to shift angle by 90 degrees
apDisplay.printWarning("|def1| > |def2|, flipping defocus axes")
tempdef = self.ctfvalues['defocus1']
self.ctfvalues['defocus1'] = self.ctfvalues['defocus2']
self.ctfvalues['defocus2'] = tempdef
self.ctfvalues['angle_astigmatism'] += 90
# get astig_angle within range -90 < angle <= 90
while self.ctfvalues['angle_astigmatism'] > 90:
self.ctfvalues['angle_astigmatism'] -= 180
while self.ctfvalues['angle_astigmatism'] < -90:
self.ctfvalues['angle_astigmatism'] += 180
avgres = self.getResolution(self.ctfvalues['defocus'], raddata, PSDarray, lowerbound)
apDisplay.printColor("Final defocus values %.3e -> %.3e, %.3e; ac=%.2f, res=%.1f"
%(self.ctfvalues['defocus'], self.ctfvalues['defocus1'], self.ctfvalues['defocus2'],
self.ctfvalues['amplitude_contrast'], avgres/2.0), "green")
for i in range(10):
print "===================================="
print "PREVIOUS VALUES"
ctfdb.getBestCtfByResolution(imgdata)
print "CURRENT VALUES"
defocusratio = self.ctfvalues['defocus2']/self.ctfvalues['defocus1']
apDisplay.printColor("def1: %.2e | def2: %.2e | angle: %.1f | ampcontr %.2f | defratio %.3f"
%(self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'],
self.ctfvalues['amplitude_contrast'], defocusratio), "blue")
self.printBestValues()
#ellipratio = self.ellipseParams['a']/self.ellipseParams['b']
print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f"%(ellipratio, defocusratio, math.sqrt(defocusratio))
print "===================================="
return
def runPhasorCTF(self, imgdata, fftpath):
### reset important values
self.bestres = 1e10
self.bestellipse = None
self.bestvalues = None
self.ellipseParams = None
self.volts = imgdata['scope']['high tension']
self.wavelength = ctftools.getTEMLambda(self.volts)
## get value in meters
self.cs = apInstrument.getCsValueFromSession(self.getSessionData())*1e-3
self.ctfvalues = {
'volts': self.volts,
'wavelength': self.wavelength,
'cs': self.cs,
}
if self.params['astig'] is False:
self.maxRatio=1.3
else:
self.maxRatio=2.0
### need to get FFT file open and freq of said file
fftarray = mrc.read(fftpath).astype(numpy.float64)
self.freq = self.freqdict[fftpath]
### print message
ctfdb.getBestCtfByResolution(imgdata)
### convert resolution limit into pixel distance
fftwidth = fftarray.shape[0]
maxres = 2.0/(self.freq*fftwidth)
if maxres > self.params['reslimit']:
apDisplay.printError("Cannot get requested res %.1fA higher than max res %.1fA"
%(maxres, self.params['reslimit']))
limitwidth = int(math.ceil(2.0/(self.params['reslimit']*self.freq)))
limitwidth = primefactor.getNextEvenPrime(limitwidth)
requestres = 2.0/(self.freq*limitwidth)
if limitwidth > fftwidth:
apDisplay.printError("Cannot get requested resolution"
+(" request res %.1fA higher than max res %.1fA for new widths %d > %d"
%(requestres, maxres, limitwidth, fftwidth)))
apDisplay.printColor("Requested resolution OK: "
+(" request res %.1fA less than max res %.1fA with fft widths %d < %d"
%(requestres, maxres, limitwidth, fftwidth)), "green")
newshape = (limitwidth, limitwidth)
fftarray = imagefilter.frame_cut(fftarray, newshape)
### spacing parameters
self.mfreq = self.freq*1e10
fftwidth = min(fftarray.shape)
self.apix = 1.0/(fftwidth*self.freq)
self.ctfvalues['apix'] = self.apix
if self.params['sample'] is 'stain':
self.ctfvalues['amplitude_contrast'] = 0.14
else:
self.ctfvalues['amplitude_contrast'] = 0.07
###
# This is the start of the actual program
###
### this is either simple rotational average,
### or complex elliptical average with edge find and RANSAC
raddata, PSDarray = self.from2Dinto1D(fftarray)
lowerrad = ctftools.getCtfExtrema(self.params['maxdef'], self.mfreq, self.cs, self.volts,
self.ctfvalues['amplitude_contrast'], 1, "valley")
lowerbound = numpy.searchsorted(raddata, lowerrad[0]*self.freq)
if self.params['sample'] is 'stain':
upperbound = numpy.searchsorted(raddata, 1/8.)
else:
upperbound = numpy.searchsorted(raddata, 1/12.)
### fun way to get initial defocus estimate
fitData = self.fitLinearPlus(raddata, PSDarray)
#lowessFit = lowess.lowess(raddata**2, PSDarray, smoothing=0.6666)
if self.debug is True:
from matplotlib import pyplot
pyplot.clf()
pyplot.plot(raddata**2, PSDarray)
pyplot.plot(raddata**2, fitData)
pyplot.show()
flatPSDarray = PSDarray-fitData
flatPSDarray /= numpy.abs(flatPSDarray[lowerbound:upperbound]).max()
defocus = self.gridSearch(raddata, flatPSDarray, lowerbound)
#defocus = findroots.estimateDefocus(raddata[lowerbound:upperbound], flatPSDarray[lowerbound:upperbound],
# cs=self.cs, wavelength=self.wavelength, amp_con=self.ctfvalues['amplitude_contrast'],
# mindef=self.params['mindef'], maxdef=self.params['maxdef'], volts=self.volts)
amplitudecontrast = sinefit.refineAmplitudeContrast(raddata[lowerbound:upperbound]*1e10, defocus,
flatPSDarray[lowerbound:upperbound], self.ctfvalues['cs'], self.wavelength, msg=self.debug)
if amplitudecontrast is not None:
print "amplitudecontrast", amplitudecontrast
self.ctfvalues['amplitude_contrast'] = amplitudecontrast
defocus = self.defocusLoop(defocus, raddata, flatPSDarray, lowerbound, upperbound)
#lowerrad = ctftools.getCtfExtrema(defocus, self.mfreq, self.cs, self.volts,
# self.ctfvalues['amplitude_contrast'], 1, "valley")
#lowerbound = numpy.searchsorted(raddata, lowerrad[0]*self.freq)
normPSDarray = self.fullTriSectionNormalize(raddata, PSDarray, defocus)
defocus = self.defocusLoop(defocus, raddata, normPSDarray, lowerbound, upperbound)
self.findEllipseLoop(fftarray, lowerbound, upperbound)
self.refineEllipseLoop(fftarray, lowerbound, upperbound)
##==================================
## FINISH UP
##==================================
apDisplay.printColor("Finishing up using best found CTF values", "blue")
self.printBestValues()
if self.bestvalues is None:
apDisplay.printColor("No CTF estimate found for this image", "red")
self.badprocess = True
return
### take best values and use them
self.ctfvalues = self.bestvalues
self.ellipseParams = self.bestellipse
### stupid fix, get value in millimeters
self.ctfvalues['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
### translate ellipse into ctf values
if self.ellipseParams is not None:
self.ctfvalues['angle_astigmatism'] = math.degrees(self.ellipseParams['alpha'])
ellipratio = self.ellipseParams['a']/self.ellipseParams['b']
phi = math.asin(self.ctfvalues['amplitude_contrast'])
#note: a > b then def1 < def2
#major axis
self.ctfvalues['defocus1'] = self.ctfvalues['defocus']/ellipratio
#minor axis
self.ctfvalues['defocus2'] = self.ctfvalues['defocus']*ellipratio
defdiff = 1.0 - 2*self.ctfvalues['defocus']/(self.ctfvalues['defocus1']+self.ctfvalues['defocus2'])
print "%.3e --> %.3e,%.3e"%(self.ctfvalues['defocus'],
self.ctfvalues['defocus2'], self.ctfvalues['defocus1'])
print defdiff*100
if defdiff*100 > 1:
apDisplay.printWarning("Large astigmatism")
#sys.exit(1)
else:
self.ctfvalues['angle_astigmatism'] = 0.0
self.ctfvalues['defocus1'] = self.ctfvalues['defocus']
self.ctfvalues['defocus2'] = self.ctfvalues['defocus']
if self.ctfvalues['amplitude_contrast'] < 0.0:
self.ctfvalues['amplitude_contrast'] = 0.0
if self.ctfvalues['amplitude_contrast'] > self.maxAmpCon:
self.ctfvalues['amplitude_contrast'] = self.maxAmpCon
if abs(self.ctfvalues['defocus1']) > abs(self.ctfvalues['defocus2']):
# incorrect, need to shift angle by 90 degrees
apDisplay.printWarning("|def1| > |def2|, flipping defocus axes")
tempdef = self.ctfvalues['defocus1']
self.ctfvalues['defocus1'] = self.ctfvalues['defocus2']
self.ctfvalues['defocus2'] = tempdef
self.ctfvalues['angle_astigmatism'] += 90
# get astig_angle within range -90 < angle <= 90
while self.ctfvalues['angle_astigmatism'] > 90:
self.ctfvalues['angle_astigmatism'] -= 180
while self.ctfvalues['angle_astigmatism'] < -90:
self.ctfvalues['angle_astigmatism'] += 180
avgres = self.getResolution(self.ctfvalues['defocus'], raddata, PSDarray, lowerbound)
apDisplay.printColor("Final defocus values %.3e -> %.3e, %.3e; ac=%.2f, res=%.1f"
%(self.ctfvalues['defocus'], self.ctfvalues['defocus1'], self.ctfvalues['defocus2'],
self.ctfvalues['amplitude_contrast'], avgres/2.0), "green")
for i in range(10):
print "===================================="
print "PREVIOUS VALUES"
ctfdb.getBestCtfByResolution(imgdata)
print "CURRENT VALUES"
defocusratio = self.ctfvalues['defocus2']/self.ctfvalues['defocus1']
apDisplay.printColor("def1: %.2e | def2: %.2e | angle: %.1f | ampcontr %.2f | defratio %.3f"
%(self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'],
self.ctfvalues['amplitude_contrast'], defocusratio), "blue")
self.printBestValues()
print "===================================="
return
def processImage(self, imgdata):
self.ctfvalues = {}
bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True)
apix = apDatabase.getPixelSize(imgdata)
if (not (self.params['onepass'] and self.params['zeropass'])):
maskhighpass = False
ace2inputpath = os.path.join(imgdata['session']['image path'],
imgdata['filename'] + ".mrc")
else:
maskhighpass = True
filterimg = apImage.maskHighPassFilter(imgdata['image'], apix, 1,
self.params['zeropass'],
self.params['onepass'])
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
mrc.write(filterimg, ace2inputpath)
# make sure that the image is a square
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
if dimx != dimy:
dims = [dimx, dimy]
dims.sort()
apDisplay.printMsg("resizing image: %ix%i to %ix%i" %
(dimx, dimy, dims[0], dims[0]))
mrcarray = apImage.mrcToArray(ace2inputpath, msg=False)
clippedmrc = apImage.frame_cut(mrcarray, [dims[0], dims[0]])
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
apImage.arrayToMrc(clippedmrc, ace2inputpath, msg=False)
### pad out image to speed up FFT calculations for non-standard image sizes
print "checking prime factor"
if primefactor.isGoodStack(dimx) is False:
goodsize = primefactor.getNextEvenPrime(dimx)
factor = float(goodsize) / float(dimx)
apDisplay.printMsg("padding image: %ix%i to %ix%i" %
(dimx, dimy, dimx * factor, dimy * factor))
mrcarray = apImage.mrcToArray(ace2inputpath, msg=False)
# paddedmrc = imagefun.pad(mrcarray, None, factor)
paddedmrc = apImage.frame_constant(mrcarray,
(dimx * factor, dimy * factor),
cval=mrcarray.mean())
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
apImage.arrayToMrc(paddedmrc, ace2inputpath, msg=False)
inputparams = {
'input': ace2inputpath,
'cs': self.params['cs'],
'kv': imgdata['scope']['high tension'] / 1000.0,
'apix': apix,
'binby': self.params['bin'],
}
### make standard input for ACE 2
apDisplay.printMsg("Ace2 executable: " + self.ace2exe)
commandline = (self.ace2exe + " -i " + str(inputparams['input']) +
" -b " + str(inputparams['binby']) + " -c " +
str(inputparams['cs']) + " -k " +
str(inputparams['kv']) + " -a " +
str(inputparams['apix']) + " -e " +
str(self.params['edge_b']) + "," +
str(self.params['edge_t']) + " -r " +
str(self.params['rotblur']) + "\n")
### run ace2
apDisplay.printMsg("running ace2 at " + time.asctime())
apDisplay.printColor(commandline, "purple")
t0 = time.time()
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline,
shell=True,
stderr=aceerrf,
stdout=aceoutf)
ace2proc.wait()
### check if ace2 worked
basename = os.path.basename(ace2inputpath)
imagelog = basename + ".ctf.txt"
if not os.path.isfile(imagelog) and self.stats['count'] <= 1:
### ace2 always crashes on first image??? .fft_wisdom file??
time.sleep(1)
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline,
shell=True,
stderr=aceerrf,
stdout=aceoutf)
ace2proc.wait()
if self.params['verbose'] is False:
aceoutf.close()
aceerrf.close()
if not os.path.isfile(imagelog):
lddcmd = "ldd " + self.ace2exe
lddproc = subprocess.Popen(lddcmd, shell=True)
lddproc.wait()
apDisplay.printError("ace2 did not run")
apDisplay.printMsg("ace2 completed in " +
apDisplay.timeString(time.time() - t0))
### parse log file
self.ctfvalues = {
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
}
logf = open(imagelog, "r")
apDisplay.printMsg("reading log file %s" % (imagelog))
for line in logf:
sline = line.strip()
if re.search("^Final Defocus: ", sline):
### old ACE2
apDisplay.printError(
"This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now"
)
#parts = sline.split()
#self.ctfvalues['defocus1'] = float(parts[2])
#self.ctfvalues['defocus2'] = float(parts[3])
### convert to degrees
#self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4]))
elif re.search("^Final Defocus \(m,m,deg\):", sline):
### new ACE2
apDisplay.printMsg("Reading new ACE2 defocus")
parts = sline.split()
#print parts
self.ctfvalues['defocus1'] = float(parts[3])
self.ctfvalues['defocus2'] = float(parts[4])
# ace2 defines negative angle from +x toward +y
self.ctfvalues['angle_astigmatism'] = -float(parts[5])
elif re.search("^Amplitude Contrast:", sline):
parts = sline.split()
self.ctfvalues['amplitude_contrast'] = float(parts[2])
elif re.search("^Confidence:", sline):
parts = sline.split()
self.ctfvalues['confidence'] = float(parts[1])
self.ctfvalues['confidence_d'] = float(parts[1])
logf.close()
### summary stats
apDisplay.printMsg("============")
avgdf = (self.ctfvalues['defocus1'] + self.ctfvalues['defocus2']) / 2.0
ampconst = 100.0 * self.ctfvalues['amplitude_contrast']
pererror = 100.0 * (self.ctfvalues['defocus1'] -
self.ctfvalues['defocus2']) / avgdf
apDisplay.printMsg(
"Defocus: %.3f x %.3f um (%.2f percent astigmatism)" %
(self.ctfvalues['defocus1'] * 1.0e6,
self.ctfvalues['defocus2'] * 1.0e6, pererror))
apDisplay.printMsg("Angle astigmatism: %.2f degrees" %
(self.ctfvalues['angle_astigmatism']))
apDisplay.printMsg("Amplitude contrast: %.2f percent" % (ampconst))
apDisplay.printColor(
"Final confidence: %.3f" % (self.ctfvalues['confidence']), 'cyan')
### double check that the values are reasonable
if avgdf > self.params['maxdefocus'] or avgdf < self.params[
'mindefocus']:
apDisplay.printWarning(
"bad defocus estimate, not committing values to database")
self.badprocess = True
if ampconst < 0.0 or ampconst > 80.0:
apDisplay.printWarning(
"bad amplitude contrast, not committing values to database")
self.badprocess = True
if self.ctfvalues['confidence'] < 0.2:
apDisplay.printWarning(
"bad confidence value, not committing values to database")
self.badprocess = True
## create power spectra jpeg
mrcfile = imgdata['filename'] + ".mrc.edge.mrc"
if os.path.isfile(mrcfile):
jpegfile = os.path.join(
self.powerspecdir,
apDisplay.short(imgdata['filename']) + ".jpg")
ps = apImage.mrcToArray(mrcfile, msg=False)
c = numpy.array(ps.shape) / 2.0
ps[c[0] - 0, c[1] - 0] = ps.mean()
ps[c[0] - 1, c[1] - 0] = ps.mean()
ps[c[0] - 0, c[1] - 1] = ps.mean()
ps[c[0] - 1, c[1] - 1] = ps.mean()
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
ps = numpy.log(ps + 1.0)
ps = (ps - ps.mean()) / ps.std()
cutoff = -2.0 * ps.min()
ps = numpy.where(ps > cutoff, cutoff, ps)
cutoff = ps.mean()
ps = numpy.where(ps < cutoff, cutoff, ps)
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
apImage.arrayToJpeg(ps, jpegfile, msg=False)
apFile.removeFile(mrcfile)
self.ctfvalues['graph3'] = jpegfile
otherfiles = glob.glob(imgdata['filename'] + ".*.txt")
### remove extra debugging files
for filename in otherfiles:
if filename[-9:] == ".norm.txt":
continue
elif filename[-8:] == ".ctf.txt":
continue
else:
apFile.removeFile(filename)
if maskhighpass and os.path.isfile(ace2inputpath):
apFile.removeFile(ace2inputpath)
return