def trimPowerSpectraToOuterResolution(powerspec, outerresolution, freq): """ freq and outerresolution must have same units (e.g., Anstroms or meters) resolution = (# columns) * apix / (pixel distance from center) therefore: pixel distance from center = (# columns) * apix / resolution """ if debug is True: print "trimPowerSpectraToOuterResolution()" imagewidth = powerspec.shape[0] initmaxres = 2.0 / (freq * imagewidth) if debug is True: print "__Image shape %d x %d" % (powerspec.shape[0], powerspec.shape[1]) print "__Frequeny %.3e" % (freq) print "__Resolution request %.3f" % (outerresolution) print "__Init max resolution %.3f" % (initmaxres) if initmaxres > outerresolution: apDisplay.printWarning( "Requested resolution (%.3f) is not available (%.3f)" % (outerresolution, initmaxres)) outerresolution = initmaxres pixellimitradius = min(powerspec.shape[0], powerspec.shape[1]) / 2 else: pixellimitradius = int(math.ceil(1. / (freq * outerresolution))) goodpixellimitradius = primefactor.getNextEvenPrime(pixellimitradius) # If outerresolution input is larger than initmaxres, the way goodpixellimitradius # is calculated above may make the result larger than half the image dimension. while goodpixellimitradius > min(powerspec.shape[0], powerspec.shape[1]) / 2: pixellimitradius -= 2 if debug is True: print "__Recalculate pixel limit dimension with: ", pixellimitradius goodpixellimitradius = primefactor.getNextEvenPrime(pixellimitradius) finalres = 1. / (freq * goodpixellimitradius) if debug is True: print "__Pixel limit dimension: ", goodpixellimitradius print "__Final max resolution %.3f" % (finalres) ### convert to diameter and trim newshape = (goodpixellimitradius * 2, goodpixellimitradius * 2) if debug is True: print "__Trimming image" trimpowerspec = imagefilter.frame_cut(powerspec, newshape) if newshape != trimpowerspec.shape: apDisplay.printError( "shape mismatch for frame_cut (%d,%d) --> (%d,%d) = (%d,%d)" % (powerspec.shape[0], powerspec.shape[1], newshape[0], newshape[1], trimpowerspec.shape[0], trimpowerspec.shape[1])) if debug is True: print "original image size %d x %d" % (powerspec.shape) print "trimmed image size %d x %d" % (trimpowerspec.shape) return trimpowerspec
def trimPowerSpectraToOuterResolution(powerspec, outerresolution, freq): """ freq and outerresolution must have same units (e.g., Anstroms or meters) resolution = (# columns) * apix / (pixel distance from center) therefore: pixel distance from center = (# columns) * apix / resolution """ if debug is True: print "trimPowerSpectraToOuterResolution()" imagewidth = powerspec.shape[0] initmaxres = 2.0/(freq*imagewidth) if debug is True: print "__Image shape %d x %d"%(powerspec.shape[0], powerspec.shape[1]) print "__Frequeny %.3e"%(freq) print "__Resolution request %.3f"%(outerresolution) print "__Init max resolution %.3f"%(initmaxres) if initmaxres > outerresolution: apDisplay.printWarning("Requested resolution (%.3f) is not available (%.3f)" %(outerresolution, initmaxres)) outerresolution = initmaxres pixellimitradius = min(powerspec.shape[0],powerspec.shape[1]) / 2 else: pixellimitradius = int(math.ceil(1./(freq * outerresolution))) goodpixellimitradius = primefactor.getNextEvenPrime(pixellimitradius) # If outerresolution input is larger than initmaxres, the way goodpixellimitradius # is calculated above may make the result larger than half the image dimension. while goodpixellimitradius > min(powerspec.shape[0],powerspec.shape[1])/2: pixellimitradius -= 2 if debug is True: print "__Recalculate pixel limit dimension with: ", pixellimitradius goodpixellimitradius = primefactor.getNextEvenPrime(pixellimitradius) finalres = 1./(freq * goodpixellimitradius) if debug is True: print "__Pixel limit dimension: ", goodpixellimitradius print "__Final max resolution %.3f"%(finalres) ### convert to diameter and trim newshape = (goodpixellimitradius*2, goodpixellimitradius*2) if debug is True: print "__Trimming image" trimpowerspec = imagefilter.frame_cut(powerspec, newshape) if newshape != trimpowerspec.shape: apDisplay.printError("shape mismatch for frame_cut (%d,%d) --> (%d,%d) = (%d,%d)" %(powerspec.shape[0],powerspec.shape[1], newshape[0],newshape[1], trimpowerspec.shape[0],trimpowerspec.shape[1])) if debug is True: print "original image size %d x %d"%(powerspec.shape) print "trimmed image size %d x %d"%(trimpowerspec.shape) return trimpowerspec
def runRefineCTF(self, imgdata, fftpath): ### reset important values self.bestres = 1e10 self.bestellipse = None self.bestvalues = {} self.volts = imgdata['scope']['high tension'] self.wavelength = ctftools.getTEMLambda(self.volts) ## get value in meters self.cs = apInstrument.getCsValueFromSession( self.getSessionData()) * 1e-3 self.ctfvalues = { 'volts': self.volts, 'wavelength': self.wavelength, 'cs': self.cs, } ### need to get FFT file open and freq of said file fftarray = mrc.read(fftpath).astype(numpy.float64) self.freq = self.freqdict[fftpath] ### convert resolution limit into pixel distance fftwidth = fftarray.shape[0] maxres = 2.0 / (self.freq * fftwidth) if maxres > self.params['reslimit']: apDisplay.printWarning( "Cannot get requested res %.1fA higher than max Nyquist resolution %.1fA" % (self.params['reslimit'], maxres)) self.params['reslimit'] = math.ceil(maxres) limitwidth = int(math.ceil(2.0 / (self.params['reslimit'] * self.freq))) limitwidth = primefactor.getNextEvenPrime(limitwidth) requestres = 2.0 / (self.freq * limitwidth) if limitwidth > fftwidth: apDisplay.printError("Cannot get requested resolution" + ( " request res %.1fA higher than max res %.1fA for new widths %d > %d" % (requestres, maxres, limitwidth, fftwidth))) apDisplay.printColor( "Requested resolution OK: " + (" request res %.1fA less than max res %.1fA with fft widths %d < %d" % (requestres, maxres, limitwidth, fftwidth)), "green") newshape = (limitwidth, limitwidth) fftarray = imagefilter.frame_cut(fftarray, newshape) ### spacing parameters self.mfreq = self.freq * 1e10 fftwidth = min(fftarray.shape) self.apix = 1.0 / (fftwidth * self.freq) self.ctfvalues['apix'] = self.apix ### print message bestDbValues = ctfdb.getBestCtfByResolution(imgdata) if bestDbValues is None: apDisplay.printColor( "SKIPPING: No CTF values for image %s" % (apDisplay.short(imgdata['filename'])), "red") self.badprocess = True return ### skip if resolution > 90. if bestDbValues['resolution_50_percent'] > 90.: apDisplay.printColor( "SKIPPING: No decent CTF values for image %s" % (apDisplay.short(imgdata['filename'])), "yellow") self.badprocess = True return self.ctfvalues['amplitude_contrast'] = bestDbValues[ 'amplitude_contrast'] lowerrad1 = ctftools.getCtfExtrema( bestDbValues['defocus1'], self.mfreq, self.cs, self.volts, self.ctfvalues['amplitude_contrast'], 1, "valley") lowerrad2 = ctftools.getCtfExtrema( bestDbValues['defocus2'], self.mfreq, self.cs, self.volts, self.ctfvalues['amplitude_contrast'], 1, "valley") meanRad = (lowerrad1[0] + lowerrad2[0]) / 2.0 self.ellipseParams = { 'a': lowerrad1[0], 'b': lowerrad2[0], 'alpha': math.radians(bestDbValues['angle_astigmatism']), } ellipratio = self.ellipseParams['a'] / self.ellipseParams['b'] defratio = bestDbValues['defocus2'] / bestDbValues['defocus1'] print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f" % (ellipratio, defratio, math.sqrt(defratio)) self.bestvalues['defocus'] = (bestDbValues['defocus1'] + bestDbValues['defocus2']) / 2.0 raddata, PSDarray = self.from2Dinto1D(fftarray) lowerbound = numpy.searchsorted(raddata, meanRad * self.freq) upperbound = numpy.searchsorted(raddata, 1 / 10.) ### # This is the start of the actual program ### self.refineEllipseLoop(fftarray, lowerbound, upperbound) ##================================== ## FINISH UP ##================================== apDisplay.printColor("Finishing up using best found CTF values", "blue") self.printBestValues() ### take best values and use them self.ctfvalues = self.bestvalues self.ellipseParams = self.bestellipse ### stupid fix, get value in millimeters self.ctfvalues['cs'] = apInstrument.getCsValueFromSession( self.getSessionData()) ### translate ellipse into ctf values if self.ellipseParams is not None: self.ctfvalues['angle_astigmatism'] = math.degrees( self.ellipseParams['alpha']) ellipratio = self.ellipseParams['a'] / self.ellipseParams['b'] phi = math.asin(self.ctfvalues['amplitude_contrast']) #note: a > b then def1 < def2 #major axis self.ctfvalues['defocus1'] = self.ctfvalues['defocus'] / ellipratio #minor axis self.ctfvalues['defocus2'] = self.ctfvalues['defocus'] * ellipratio defdiff = 1.0 - 2 * self.ctfvalues['defocus'] / ( self.ctfvalues['defocus1'] + self.ctfvalues['defocus2']) print "%.3e --> %.3e,%.3e" % (self.ctfvalues['defocus'], self.ctfvalues['defocus2'], self.ctfvalues['defocus1']) print defdiff * 100 if defdiff * 100 > 1: apDisplay.printWarning("Large astigmatism") #sys.exit(1) else: ellipratio = 1.0 self.ctfvalues['angle_astigmatism'] = 0.0 self.ctfvalues['defocus1'] = self.ctfvalues['defocus'] self.ctfvalues['defocus2'] = self.ctfvalues['defocus'] self.badprocess = True return try: if self.ctfvalues['amplitude_contrast'] < self.minAmpCon: self.ctfvalues['amplitude_contrast'] = self.minAmpCon if self.ctfvalues['amplitude_contrast'] > self.maxAmpCon: self.ctfvalues['amplitude_contrast'] = self.maxAmpCon except KeyError: pass if abs(self.ctfvalues['defocus1']) > abs(self.ctfvalues['defocus2']): # incorrect, need to shift angle by 90 degrees apDisplay.printWarning("|def1| > |def2|, flipping defocus axes") tempdef = self.ctfvalues['defocus1'] self.ctfvalues['defocus1'] = self.ctfvalues['defocus2'] self.ctfvalues['defocus2'] = tempdef self.ctfvalues['angle_astigmatism'] += 90 # get astig_angle within range -90 < angle <= 90 while self.ctfvalues['angle_astigmatism'] > 90: self.ctfvalues['angle_astigmatism'] -= 180 while self.ctfvalues['angle_astigmatism'] < -90: self.ctfvalues['angle_astigmatism'] += 180 avgres = self.getResolution(self.ctfvalues['defocus'], raddata, PSDarray, lowerbound) apDisplay.printColor( "Final defocus values %.3e -> %.3e, %.3e; ac=%.2f, res=%.1f" % (self.ctfvalues['defocus'], self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['amplitude_contrast'], avgres / 2.0), "green") for i in range(10): print "====================================" print "PREVIOUS VALUES" ctfdb.getBestCtfByResolution(imgdata) print "CURRENT VALUES" defocusratio = self.ctfvalues['defocus2'] / self.ctfvalues['defocus1'] apDisplay.printColor( "def1: %.2e | def2: %.2e | angle: %.1f | ampcontr %.2f | defratio %.3f" % (self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'], self.ctfvalues['amplitude_contrast'], defocusratio), "blue") self.printBestValues() #ellipratio = self.ellipseParams['a']/self.ellipseParams['b'] print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f" % ( ellipratio, defocusratio, math.sqrt(defocusratio)) print "====================================" return
def processImage(self, imgdata): self.ctfvalues = {} bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True) apix = apDatabase.getPixelSize(imgdata) if (not (self.params['onepass'] and self.params['zeropass'])): maskhighpass = False ace2inputpath = os.path.join(imgdata['session']['image path'],imgdata['filename']+".mrc") else: maskhighpass = True filterimg = apImage.maskHighPassFilter(imgdata['image'],apix,1,self.params['zeropass'],self.params['onepass']) ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc") mrc.write(filterimg,ace2inputpath) # make sure that the image is a square dimx = imgdata['camera']['dimension']['x'] dimy = imgdata['camera']['dimension']['y'] if dimx != dimy: dims = [dimx,dimy] dims.sort() apDisplay.printMsg("resizing image: %ix%i to %ix%i" % (dimx,dimy,dims[0],dims[0])) mrcarray = apImage.mrcToArray(ace2inputpath,msg=False) clippedmrc = apImage.frame_cut(mrcarray,[dims[0],dims[0]]) ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc") apImage.arrayToMrc(clippedmrc,ace2inputpath,msg=False) ### pad out image to speed up FFT calculations for non-standard image sizes print "checking prime factor" if primefactor.isGoodStack(dimx) is False: goodsize = primefactor.getNextEvenPrime(dimx) factor = float(goodsize) / float(dimx) apDisplay.printMsg("padding image: %ix%i to %ix%i" % (dimx,dimy,dimx*factor,dimy*factor)) mrcarray = apImage.mrcToArray(ace2inputpath,msg=False) # paddedmrc = imagefun.pad(mrcarray, None, factor) paddedmrc = apImage.frame_constant(mrcarray, (dimx*factor,dimy*factor), cval=mrcarray.mean()) ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc") apImage.arrayToMrc(paddedmrc,ace2inputpath,msg=False) inputparams = { 'input': ace2inputpath, 'cs': self.params['cs'], 'kv': imgdata['scope']['high tension']/1000.0, 'apix': apix, 'binby': self.params['bin'], } ### make standard input for ACE 2 apDisplay.printMsg("Ace2 executable: "+self.ace2exe) commandline = ( self.ace2exe + " -i " + str(inputparams['input']) + " -b " + str(inputparams['binby']) + " -c " + str(inputparams['cs']) + " -k " + str(inputparams['kv']) + " -a " + str(inputparams['apix']) + " -e " + str(self.params['edge_b'])+","+str(self.params['edge_t']) + " -r " + str(self.params['rotblur']) + "\n" ) ### run ace2 apDisplay.printMsg("running ace2 at "+time.asctime()) apDisplay.printColor(commandline, "purple") t0 = time.time() if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() ### check if ace2 worked basename = os.path.basename(ace2inputpath) imagelog = basename+".ctf.txt" if not os.path.isfile(imagelog) and self.stats['count'] <= 1: ### ace2 always crashes on first image??? .fft_wisdom file?? time.sleep(1) if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() if self.params['verbose'] is False: aceoutf.close() aceerrf.close() if not os.path.isfile(imagelog): lddcmd = "ldd "+self.ace2exe lddproc = subprocess.Popen(lddcmd, shell=True) lddproc.wait() apDisplay.printError("ace2 did not run") apDisplay.printMsg("ace2 completed in " + apDisplay.timeString(time.time()-t0)) ### parse log file self.ctfvalues = { 'cs': self.params['cs'], 'volts': imgdata['scope']['high tension'], } logf = open(imagelog, "r") apDisplay.printMsg("reading log file %s"%(imagelog)) for line in logf: sline = line.strip() if re.search("^Final Defocus: ", sline): ### old ACE2 apDisplay.printError("This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now") #parts = sline.split() #self.ctfvalues['defocus1'] = float(parts[2]) #self.ctfvalues['defocus2'] = float(parts[3]) ### convert to degrees #self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4])) elif re.search("^Final Defocus \(m,m,deg\):", sline): ### new ACE2 apDisplay.printMsg("Reading new ACE2 defocus") parts = sline.split() #print parts self.ctfvalues['defocus1'] = float(parts[3]) self.ctfvalues['defocus2'] = float(parts[4]) # ace2 defines negative angle from +x toward +y self.ctfvalues['angle_astigmatism'] = -float(parts[5]) elif re.search("^Amplitude Contrast:",sline): parts = sline.split() self.ctfvalues['amplitude_contrast'] = float(parts[2]) elif re.search("^Confidence:",sline): parts = sline.split() self.ctfvalues['confidence'] = float(parts[1]) self.ctfvalues['confidence_d'] = float(parts[1]) logf.close() ### summary stats apDisplay.printMsg("============") avgdf = (self.ctfvalues['defocus1']+self.ctfvalues['defocus2'])/2.0 ampconst = 100.0*self.ctfvalues['amplitude_contrast'] pererror = 100.0 * (self.ctfvalues['defocus1']-self.ctfvalues['defocus2']) / avgdf apDisplay.printMsg("Defocus: %.3f x %.3f um (%.2f percent astigmatism)"% (self.ctfvalues['defocus1']*1.0e6, self.ctfvalues['defocus2']*1.0e6, pererror )) apDisplay.printMsg("Angle astigmatism: %.2f degrees"%(self.ctfvalues['angle_astigmatism'])) apDisplay.printMsg("Amplitude contrast: %.2f percent"%(ampconst)) apDisplay.printColor("Final confidence: %.3f"%(self.ctfvalues['confidence']),'cyan') ### double check that the values are reasonable if avgdf > self.params['maxdefocus'] or avgdf < self.params['mindefocus']: apDisplay.printWarning("bad defocus estimate, not committing values to database") self.badprocess = True if ampconst < 0.0 or ampconst > 80.0: apDisplay.printWarning("bad amplitude contrast, not committing values to database") self.badprocess = True if self.ctfvalues['confidence'] < 0.2: apDisplay.printWarning("bad confidence value, not committing values to database") self.badprocess = True ## create power spectra jpeg mrcfile = imgdata['filename']+".mrc.edge.mrc" if os.path.isfile(mrcfile): jpegfile = os.path.join(self.powerspecdir, apDisplay.short(imgdata['filename'])+".jpg") ps = apImage.mrcToArray(mrcfile,msg=False) c = numpy.array(ps.shape)/2.0 ps[c[0]-0,c[1]-0] = ps.mean() ps[c[0]-1,c[1]-0] = ps.mean() ps[c[0]-0,c[1]-1] = ps.mean() ps[c[0]-1,c[1]-1] = ps.mean() #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) ps = numpy.log(ps+1.0) ps = (ps-ps.mean())/ps.std() cutoff = -2.0*ps.min() ps = numpy.where(ps > cutoff, cutoff, ps) cutoff = ps.mean() ps = numpy.where(ps < cutoff, cutoff, ps) #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) apImage.arrayToJpeg(ps, jpegfile, msg=False) apFile.removeFile(mrcfile) self.ctfvalues['graph3'] = jpegfile otherfiles = glob.glob(imgdata['filename']+".*.txt") ### remove extra debugging files for filename in otherfiles: if filename[-9:] == ".norm.txt": continue elif filename[-8:] == ".ctf.txt": continue else: apFile.removeFile(filename) if maskhighpass and os.path.isfile(ace2inputpath): apFile.removeFile(ace2inputpath) return
def runRefineCTF(self, imgdata, fftpath): ### reset important values self.bestres = 1e10 self.bestellipse = None self.bestvalues = {} self.volts = imgdata['scope']['high tension'] self.wavelength = ctftools.getTEMLambda(self.volts) ## get value in meters self.cs = apInstrument.getCsValueFromSession(self.getSessionData())*1e-3 self.ctfvalues = { 'volts': self.volts, 'wavelength': self.wavelength, 'cs': self.cs, } ### need to get FFT file open and freq of said file fftarray = mrc.read(fftpath).astype(numpy.float64) self.freq = self.freqdict[fftpath] ### convert resolution limit into pixel distance fftwidth = fftarray.shape[0] maxres = 2.0/(self.freq*fftwidth) if maxres > self.params['reslimit']: apDisplay.printError("Cannot get requested res %.1fA higher than max res %.1fA" %(maxres, self.params['reslimit'])) limitwidth = int(math.ceil(2.0/(self.params['reslimit']*self.freq))) limitwidth = primefactor.getNextEvenPrime(limitwidth) requestres = 2.0/(self.freq*limitwidth) if limitwidth > fftwidth: apDisplay.printError("Cannot get requested resolution" +(" request res %.1fA higher than max res %.1fA for new widths %d > %d" %(requestres, maxres, limitwidth, fftwidth))) apDisplay.printColor("Requested resolution OK: " +(" request res %.1fA less than max res %.1fA with fft widths %d < %d" %(requestres, maxres, limitwidth, fftwidth)), "green") newshape = (limitwidth, limitwidth) fftarray = imagefilter.frame_cut(fftarray, newshape) ### spacing parameters self.mfreq = self.freq*1e10 fftwidth = min(fftarray.shape) self.apix = 1.0/(fftwidth*self.freq) self.ctfvalues['apix'] = self.apix ### print message bestDbValues = ctfdb.getBestCtfByResolution(imgdata) if bestDbValues is None: apDisplay.printColor("SKIPPING: No CTF values for image %s" %(apDisplay.short(imgdata['filename'])), "red") self.badprocess = True return ### skip if resolution > 90. if bestDbValues['resolution_50_percent'] > 90.: apDisplay.printColor("SKIPPING: No decent CTF values for image %s" %(apDisplay.short(imgdata['filename'])), "yellow") self.badprocess = True return self.ctfvalues['amplitude_contrast'] = bestDbValues['amplitude_contrast'] lowerrad1 = ctftools.getCtfExtrema(bestDbValues['defocus1'], self.mfreq, self.cs, self.volts, self.ctfvalues['amplitude_contrast'], 1, "valley") lowerrad2 = ctftools.getCtfExtrema(bestDbValues['defocus2'], self.mfreq, self.cs, self.volts, self.ctfvalues['amplitude_contrast'], 1, "valley") meanRad = (lowerrad1[0] + lowerrad2[0])/2.0 self.ellipseParams = { 'a': lowerrad1[0], 'b': lowerrad2[0], 'alpha': math.radians(bestDbValues['angle_astigmatism']), } ellipratio = self.ellipseParams['a']/self.ellipseParams['b'] defratio = bestDbValues['defocus2']/bestDbValues['defocus1'] print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f"%(ellipratio, defratio, math.sqrt(defratio)) self.bestvalues['defocus'] = (bestDbValues['defocus1']+bestDbValues['defocus2'])/2.0 raddata, PSDarray = self.from2Dinto1D(fftarray) lowerbound = numpy.searchsorted(raddata, meanRad*self.freq) upperbound = numpy.searchsorted(raddata, 1/10.) ### # This is the start of the actual program ### self.refineEllipseLoop(fftarray, lowerbound, upperbound) ##================================== ## FINISH UP ##================================== apDisplay.printColor("Finishing up using best found CTF values", "blue") self.printBestValues() ### take best values and use them self.ctfvalues = self.bestvalues self.ellipseParams = self.bestellipse ### stupid fix, get value in millimeters self.ctfvalues['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) ### translate ellipse into ctf values if self.ellipseParams is not None: self.ctfvalues['angle_astigmatism'] = math.degrees(self.ellipseParams['alpha']) ellipratio = self.ellipseParams['a']/self.ellipseParams['b'] phi = math.asin(self.ctfvalues['amplitude_contrast']) #note: a > b then def1 < def2 #major axis self.ctfvalues['defocus1'] = self.ctfvalues['defocus']/ellipratio #minor axis self.ctfvalues['defocus2'] = self.ctfvalues['defocus']*ellipratio defdiff = 1.0 - 2*self.ctfvalues['defocus']/(self.ctfvalues['defocus1']+self.ctfvalues['defocus2']) print "%.3e --> %.3e,%.3e"%(self.ctfvalues['defocus'], self.ctfvalues['defocus2'], self.ctfvalues['defocus1']) print defdiff*100 if defdiff*100 > 1: apDisplay.printWarning("Large astigmatism") #sys.exit(1) else: ellipratio = 1.0 self.ctfvalues['angle_astigmatism'] = 0.0 self.ctfvalues['defocus1'] = self.ctfvalues['defocus'] self.ctfvalues['defocus2'] = self.ctfvalues['defocus'] if self.ctfvalues['amplitude_contrast'] < 0.0: self.ctfvalues['amplitude_contrast'] = 0.0 if self.ctfvalues['amplitude_contrast'] > self.maxAmpCon: self.ctfvalues['amplitude_contrast'] = self.maxAmpCon if abs(self.ctfvalues['defocus1']) > abs(self.ctfvalues['defocus2']): # incorrect, need to shift angle by 90 degrees apDisplay.printWarning("|def1| > |def2|, flipping defocus axes") tempdef = self.ctfvalues['defocus1'] self.ctfvalues['defocus1'] = self.ctfvalues['defocus2'] self.ctfvalues['defocus2'] = tempdef self.ctfvalues['angle_astigmatism'] += 90 # get astig_angle within range -90 < angle <= 90 while self.ctfvalues['angle_astigmatism'] > 90: self.ctfvalues['angle_astigmatism'] -= 180 while self.ctfvalues['angle_astigmatism'] < -90: self.ctfvalues['angle_astigmatism'] += 180 avgres = self.getResolution(self.ctfvalues['defocus'], raddata, PSDarray, lowerbound) apDisplay.printColor("Final defocus values %.3e -> %.3e, %.3e; ac=%.2f, res=%.1f" %(self.ctfvalues['defocus'], self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['amplitude_contrast'], avgres/2.0), "green") for i in range(10): print "====================================" print "PREVIOUS VALUES" ctfdb.getBestCtfByResolution(imgdata) print "CURRENT VALUES" defocusratio = self.ctfvalues['defocus2']/self.ctfvalues['defocus1'] apDisplay.printColor("def1: %.2e | def2: %.2e | angle: %.1f | ampcontr %.2f | defratio %.3f" %(self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'], self.ctfvalues['amplitude_contrast'], defocusratio), "blue") self.printBestValues() #ellipratio = self.ellipseParams['a']/self.ellipseParams['b'] print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f"%(ellipratio, defocusratio, math.sqrt(defocusratio)) print "====================================" return
def runPhasorCTF(self, imgdata, fftpath): ### reset important values self.bestres = 1e10 self.bestellipse = None self.bestvalues = None self.ellipseParams = None self.volts = imgdata['scope']['high tension'] self.wavelength = ctftools.getTEMLambda(self.volts) ## get value in meters self.cs = apInstrument.getCsValueFromSession(self.getSessionData())*1e-3 self.ctfvalues = { 'volts': self.volts, 'wavelength': self.wavelength, 'cs': self.cs, } if self.params['astig'] is False: self.maxRatio=1.3 else: self.maxRatio=2.0 ### need to get FFT file open and freq of said file fftarray = mrc.read(fftpath).astype(numpy.float64) self.freq = self.freqdict[fftpath] ### print message ctfdb.getBestCtfByResolution(imgdata) ### convert resolution limit into pixel distance fftwidth = fftarray.shape[0] maxres = 2.0/(self.freq*fftwidth) if maxres > self.params['reslimit']: apDisplay.printError("Cannot get requested res %.1fA higher than max res %.1fA" %(maxres, self.params['reslimit'])) limitwidth = int(math.ceil(2.0/(self.params['reslimit']*self.freq))) limitwidth = primefactor.getNextEvenPrime(limitwidth) requestres = 2.0/(self.freq*limitwidth) if limitwidth > fftwidth: apDisplay.printError("Cannot get requested resolution" +(" request res %.1fA higher than max res %.1fA for new widths %d > %d" %(requestres, maxres, limitwidth, fftwidth))) apDisplay.printColor("Requested resolution OK: " +(" request res %.1fA less than max res %.1fA with fft widths %d < %d" %(requestres, maxres, limitwidth, fftwidth)), "green") newshape = (limitwidth, limitwidth) fftarray = imagefilter.frame_cut(fftarray, newshape) ### spacing parameters self.mfreq = self.freq*1e10 fftwidth = min(fftarray.shape) self.apix = 1.0/(fftwidth*self.freq) self.ctfvalues['apix'] = self.apix if self.params['sample'] is 'stain': self.ctfvalues['amplitude_contrast'] = 0.14 else: self.ctfvalues['amplitude_contrast'] = 0.07 ### # This is the start of the actual program ### ### this is either simple rotational average, ### or complex elliptical average with edge find and RANSAC raddata, PSDarray = self.from2Dinto1D(fftarray) lowerrad = ctftools.getCtfExtrema(self.params['maxdef'], self.mfreq, self.cs, self.volts, self.ctfvalues['amplitude_contrast'], 1, "valley") lowerbound = numpy.searchsorted(raddata, lowerrad[0]*self.freq) if self.params['sample'] is 'stain': upperbound = numpy.searchsorted(raddata, 1/8.) else: upperbound = numpy.searchsorted(raddata, 1/12.) ### fun way to get initial defocus estimate fitData = self.fitLinearPlus(raddata, PSDarray) #lowessFit = lowess.lowess(raddata**2, PSDarray, smoothing=0.6666) if self.debug is True: from matplotlib import pyplot pyplot.clf() pyplot.plot(raddata**2, PSDarray) pyplot.plot(raddata**2, fitData) pyplot.show() flatPSDarray = PSDarray-fitData flatPSDarray /= numpy.abs(flatPSDarray[lowerbound:upperbound]).max() defocus = self.gridSearch(raddata, flatPSDarray, lowerbound) #defocus = findroots.estimateDefocus(raddata[lowerbound:upperbound], flatPSDarray[lowerbound:upperbound], # cs=self.cs, wavelength=self.wavelength, amp_con=self.ctfvalues['amplitude_contrast'], # mindef=self.params['mindef'], maxdef=self.params['maxdef'], volts=self.volts) amplitudecontrast = sinefit.refineAmplitudeContrast(raddata[lowerbound:upperbound]*1e10, defocus, flatPSDarray[lowerbound:upperbound], self.ctfvalues['cs'], self.wavelength, msg=self.debug) if amplitudecontrast is not None: print "amplitudecontrast", amplitudecontrast self.ctfvalues['amplitude_contrast'] = amplitudecontrast defocus = self.defocusLoop(defocus, raddata, flatPSDarray, lowerbound, upperbound) #lowerrad = ctftools.getCtfExtrema(defocus, self.mfreq, self.cs, self.volts, # self.ctfvalues['amplitude_contrast'], 1, "valley") #lowerbound = numpy.searchsorted(raddata, lowerrad[0]*self.freq) normPSDarray = self.fullTriSectionNormalize(raddata, PSDarray, defocus) defocus = self.defocusLoop(defocus, raddata, normPSDarray, lowerbound, upperbound) self.findEllipseLoop(fftarray, lowerbound, upperbound) self.refineEllipseLoop(fftarray, lowerbound, upperbound) ##================================== ## FINISH UP ##================================== apDisplay.printColor("Finishing up using best found CTF values", "blue") self.printBestValues() if self.bestvalues is None: apDisplay.printColor("No CTF estimate found for this image", "red") self.badprocess = True return ### take best values and use them self.ctfvalues = self.bestvalues self.ellipseParams = self.bestellipse ### stupid fix, get value in millimeters self.ctfvalues['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) ### translate ellipse into ctf values if self.ellipseParams is not None: self.ctfvalues['angle_astigmatism'] = math.degrees(self.ellipseParams['alpha']) ellipratio = self.ellipseParams['a']/self.ellipseParams['b'] phi = math.asin(self.ctfvalues['amplitude_contrast']) #note: a > b then def1 < def2 #major axis self.ctfvalues['defocus1'] = self.ctfvalues['defocus']/ellipratio #minor axis self.ctfvalues['defocus2'] = self.ctfvalues['defocus']*ellipratio defdiff = 1.0 - 2*self.ctfvalues['defocus']/(self.ctfvalues['defocus1']+self.ctfvalues['defocus2']) print "%.3e --> %.3e,%.3e"%(self.ctfvalues['defocus'], self.ctfvalues['defocus2'], self.ctfvalues['defocus1']) print defdiff*100 if defdiff*100 > 1: apDisplay.printWarning("Large astigmatism") #sys.exit(1) else: self.ctfvalues['angle_astigmatism'] = 0.0 self.ctfvalues['defocus1'] = self.ctfvalues['defocus'] self.ctfvalues['defocus2'] = self.ctfvalues['defocus'] if self.ctfvalues['amplitude_contrast'] < 0.0: self.ctfvalues['amplitude_contrast'] = 0.0 if self.ctfvalues['amplitude_contrast'] > self.maxAmpCon: self.ctfvalues['amplitude_contrast'] = self.maxAmpCon if abs(self.ctfvalues['defocus1']) > abs(self.ctfvalues['defocus2']): # incorrect, need to shift angle by 90 degrees apDisplay.printWarning("|def1| > |def2|, flipping defocus axes") tempdef = self.ctfvalues['defocus1'] self.ctfvalues['defocus1'] = self.ctfvalues['defocus2'] self.ctfvalues['defocus2'] = tempdef self.ctfvalues['angle_astigmatism'] += 90 # get astig_angle within range -90 < angle <= 90 while self.ctfvalues['angle_astigmatism'] > 90: self.ctfvalues['angle_astigmatism'] -= 180 while self.ctfvalues['angle_astigmatism'] < -90: self.ctfvalues['angle_astigmatism'] += 180 avgres = self.getResolution(self.ctfvalues['defocus'], raddata, PSDarray, lowerbound) apDisplay.printColor("Final defocus values %.3e -> %.3e, %.3e; ac=%.2f, res=%.1f" %(self.ctfvalues['defocus'], self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['amplitude_contrast'], avgres/2.0), "green") for i in range(10): print "====================================" print "PREVIOUS VALUES" ctfdb.getBestCtfByResolution(imgdata) print "CURRENT VALUES" defocusratio = self.ctfvalues['defocus2']/self.ctfvalues['defocus1'] apDisplay.printColor("def1: %.2e | def2: %.2e | angle: %.1f | ampcontr %.2f | defratio %.3f" %(self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'], self.ctfvalues['amplitude_contrast'], defocusratio), "blue") self.printBestValues() print "====================================" return
def processImage(self, imgdata): self.ctfvalues = {} bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True) apix = apDatabase.getPixelSize(imgdata) if (not (self.params['onepass'] and self.params['zeropass'])): maskhighpass = False ace2inputpath = os.path.join(imgdata['session']['image path'], imgdata['filename'] + ".mrc") else: maskhighpass = True filterimg = apImage.maskHighPassFilter(imgdata['image'], apix, 1, self.params['zeropass'], self.params['onepass']) ace2inputpath = os.path.join(self.params['rundir'], imgdata['filename'] + ".mrc") mrc.write(filterimg, ace2inputpath) # make sure that the image is a square dimx = imgdata['camera']['dimension']['x'] dimy = imgdata['camera']['dimension']['y'] if dimx != dimy: dims = [dimx, dimy] dims.sort() apDisplay.printMsg("resizing image: %ix%i to %ix%i" % (dimx, dimy, dims[0], dims[0])) mrcarray = apImage.mrcToArray(ace2inputpath, msg=False) clippedmrc = apImage.frame_cut(mrcarray, [dims[0], dims[0]]) ace2inputpath = os.path.join(self.params['rundir'], imgdata['filename'] + ".mrc") apImage.arrayToMrc(clippedmrc, ace2inputpath, msg=False) ### pad out image to speed up FFT calculations for non-standard image sizes print "checking prime factor" if primefactor.isGoodStack(dimx) is False: goodsize = primefactor.getNextEvenPrime(dimx) factor = float(goodsize) / float(dimx) apDisplay.printMsg("padding image: %ix%i to %ix%i" % (dimx, dimy, dimx * factor, dimy * factor)) mrcarray = apImage.mrcToArray(ace2inputpath, msg=False) # paddedmrc = imagefun.pad(mrcarray, None, factor) paddedmrc = apImage.frame_constant(mrcarray, (dimx * factor, dimy * factor), cval=mrcarray.mean()) ace2inputpath = os.path.join(self.params['rundir'], imgdata['filename'] + ".mrc") apImage.arrayToMrc(paddedmrc, ace2inputpath, msg=False) inputparams = { 'input': ace2inputpath, 'cs': self.params['cs'], 'kv': imgdata['scope']['high tension'] / 1000.0, 'apix': apix, 'binby': self.params['bin'], } ### make standard input for ACE 2 apDisplay.printMsg("Ace2 executable: " + self.ace2exe) commandline = (self.ace2exe + " -i " + str(inputparams['input']) + " -b " + str(inputparams['binby']) + " -c " + str(inputparams['cs']) + " -k " + str(inputparams['kv']) + " -a " + str(inputparams['apix']) + " -e " + str(self.params['edge_b']) + "," + str(self.params['edge_t']) + " -r " + str(self.params['rotblur']) + "\n") ### run ace2 apDisplay.printMsg("running ace2 at " + time.asctime()) apDisplay.printColor(commandline, "purple") t0 = time.time() if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() ### check if ace2 worked basename = os.path.basename(ace2inputpath) imagelog = basename + ".ctf.txt" if not os.path.isfile(imagelog) and self.stats['count'] <= 1: ### ace2 always crashes on first image??? .fft_wisdom file?? time.sleep(1) if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() if self.params['verbose'] is False: aceoutf.close() aceerrf.close() if not os.path.isfile(imagelog): lddcmd = "ldd " + self.ace2exe lddproc = subprocess.Popen(lddcmd, shell=True) lddproc.wait() apDisplay.printError("ace2 did not run") apDisplay.printMsg("ace2 completed in " + apDisplay.timeString(time.time() - t0)) ### parse log file self.ctfvalues = { 'cs': self.params['cs'], 'volts': imgdata['scope']['high tension'], } logf = open(imagelog, "r") apDisplay.printMsg("reading log file %s" % (imagelog)) for line in logf: sline = line.strip() if re.search("^Final Defocus: ", sline): ### old ACE2 apDisplay.printError( "This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now" ) #parts = sline.split() #self.ctfvalues['defocus1'] = float(parts[2]) #self.ctfvalues['defocus2'] = float(parts[3]) ### convert to degrees #self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4])) elif re.search("^Final Defocus \(m,m,deg\):", sline): ### new ACE2 apDisplay.printMsg("Reading new ACE2 defocus") parts = sline.split() #print parts self.ctfvalues['defocus1'] = float(parts[3]) self.ctfvalues['defocus2'] = float(parts[4]) # ace2 defines negative angle from +x toward +y self.ctfvalues['angle_astigmatism'] = -float(parts[5]) elif re.search("^Amplitude Contrast:", sline): parts = sline.split() self.ctfvalues['amplitude_contrast'] = float(parts[2]) elif re.search("^Confidence:", sline): parts = sline.split() self.ctfvalues['confidence'] = float(parts[1]) self.ctfvalues['confidence_d'] = float(parts[1]) logf.close() ### summary stats apDisplay.printMsg("============") avgdf = (self.ctfvalues['defocus1'] + self.ctfvalues['defocus2']) / 2.0 ampconst = 100.0 * self.ctfvalues['amplitude_contrast'] pererror = 100.0 * (self.ctfvalues['defocus1'] - self.ctfvalues['defocus2']) / avgdf apDisplay.printMsg( "Defocus: %.3f x %.3f um (%.2f percent astigmatism)" % (self.ctfvalues['defocus1'] * 1.0e6, self.ctfvalues['defocus2'] * 1.0e6, pererror)) apDisplay.printMsg("Angle astigmatism: %.2f degrees" % (self.ctfvalues['angle_astigmatism'])) apDisplay.printMsg("Amplitude contrast: %.2f percent" % (ampconst)) apDisplay.printColor( "Final confidence: %.3f" % (self.ctfvalues['confidence']), 'cyan') ### double check that the values are reasonable if avgdf > self.params['maxdefocus'] or avgdf < self.params[ 'mindefocus']: apDisplay.printWarning( "bad defocus estimate, not committing values to database") self.badprocess = True if ampconst < 0.0 or ampconst > 80.0: apDisplay.printWarning( "bad amplitude contrast, not committing values to database") self.badprocess = True if self.ctfvalues['confidence'] < 0.2: apDisplay.printWarning( "bad confidence value, not committing values to database") self.badprocess = True ## create power spectra jpeg mrcfile = imgdata['filename'] + ".mrc.edge.mrc" if os.path.isfile(mrcfile): jpegfile = os.path.join( self.powerspecdir, apDisplay.short(imgdata['filename']) + ".jpg") ps = apImage.mrcToArray(mrcfile, msg=False) c = numpy.array(ps.shape) / 2.0 ps[c[0] - 0, c[1] - 0] = ps.mean() ps[c[0] - 1, c[1] - 0] = ps.mean() ps[c[0] - 0, c[1] - 1] = ps.mean() ps[c[0] - 1, c[1] - 1] = ps.mean() #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) ps = numpy.log(ps + 1.0) ps = (ps - ps.mean()) / ps.std() cutoff = -2.0 * ps.min() ps = numpy.where(ps > cutoff, cutoff, ps) cutoff = ps.mean() ps = numpy.where(ps < cutoff, cutoff, ps) #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) apImage.arrayToJpeg(ps, jpegfile, msg=False) apFile.removeFile(mrcfile) self.ctfvalues['graph3'] = jpegfile otherfiles = glob.glob(imgdata['filename'] + ".*.txt") ### remove extra debugging files for filename in otherfiles: if filename[-9:] == ".norm.txt": continue elif filename[-8:] == ".ctf.txt": continue else: apFile.removeFile(filename) if maskhighpass and os.path.isfile(ace2inputpath): apFile.removeFile(ace2inputpath) return