def scaleTemplate(templatearray, scalefactor=1.0, boxsize=None): if(templatearray.shape[0] != templatearray.shape[1]): apDisplay.printWarning("template shape is NOT square, this may cause errors") if abs(scalefactor - 1.0) > 0.01: apDisplay.printMsg("scaling template by a factor of "+str(scalefactor)) templatearray = apImage.scaleImage(templatearray, scalefactor) #make sure the box size is divisible by 16 if boxsize is not None or (templatearray.shape[0] % 16 != 0): edgeavg = apImage.meanEdgeValue(templatearray) origsize = templatearray.shape[0] if boxsize is None: # minimal padisize is 16 padsize = max(int(math.floor(float(origsize)/16)*16),16) else: padsize = boxsize padshape = numpy.array([padsize,padsize]) apDisplay.printMsg("changing box size from "+str(origsize)+" to "+str(padsize)) if origsize > padsize: #shrink image templatearray = apImage.frame_cut(templatearray, padshape) else: #grow image templatearray = apImage.frame_constant(templatearray, padshape, cval=edgeavg) if templatearray.shape[0] < 20 or templatearray.shape[1] < 20: apDisplay.printWarning("template is only "+str(templatearray.shape[0])+" pixels wide\n"+\ " and may only correlation noise in the image") return templatearray
def scaleTemplate(templatearray, scalefactor=1.0, boxsize=None): if (templatearray.shape[0] != templatearray.shape[1]): apDisplay.printWarning( "template shape is NOT square, this may cause errors") if abs(scalefactor - 1.0) > 0.01: apDisplay.printMsg("scaling template by a factor of " + str(scalefactor)) templatearray = apImage.scaleImage(templatearray, scalefactor) #make sure the box size is divisible by 16 if boxsize is not None or (templatearray.shape[0] % 16 != 0): edgeavg = apImage.meanEdgeValue(templatearray) origsize = templatearray.shape[0] if boxsize is None: # minimal padisize is 16 padsize = max(int(math.floor(float(origsize) / 16) * 16), 16) else: padsize = boxsize padshape = numpy.array([padsize, padsize]) apDisplay.printMsg("changing box size from " + str(origsize) + " to " + str(padsize)) if origsize > padsize: #shrink image templatearray = apImage.frame_cut(templatearray, padshape) else: #grow image templatearray = apImage.frame_constant(templatearray, padshape, cval=edgeavg) if templatearray.shape[0] < 20 or templatearray.shape[1] < 20: apDisplay.printWarning("template is only "+str(templatearray.shape[0])+" pixels wide\n"+\ " and may only correlation noise in the image") return templatearray
def real_fft3d(volume, *args, **kwargs): padshape = numpy.asarray(volume.shape) * 1 padvolume = apImage.frame_constant(volume, padshape, volume.mean()) fft = fftpack.fftn(padvolume, *args, **kwargs) return fft
def real_fft2d(image, *args, **kwargs): padshape = numpy.asarray(image.shape) * 1 padimage = apImage.frame_constant(image, padshape, image.mean()) fft = fftpack.fft2(padimage, *args, **kwargs) # normfft = (fft - fft.mean())/fft.std() return fft
def processImage(self, imgdata): self.ctfvalues = {} bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True) apix = apDatabase.getPixelSize(imgdata) if (not (self.params['onepass'] and self.params['zeropass'])): maskhighpass = False ace2inputpath = os.path.join(imgdata['session']['image path'],imgdata['filename']+".mrc") else: maskhighpass = True filterimg = apImage.maskHighPassFilter(imgdata['image'],apix,1,self.params['zeropass'],self.params['onepass']) ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc") mrc.write(filterimg,ace2inputpath) # make sure that the image is a square dimx = imgdata['camera']['dimension']['x'] dimy = imgdata['camera']['dimension']['y'] if dimx != dimy: dims = [dimx,dimy] dims.sort() apDisplay.printMsg("resizing image: %ix%i to %ix%i" % (dimx,dimy,dims[0],dims[0])) mrcarray = apImage.mrcToArray(ace2inputpath,msg=False) clippedmrc = apImage.frame_cut(mrcarray,[dims[0],dims[0]]) ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc") apImage.arrayToMrc(clippedmrc,ace2inputpath,msg=False) ### pad out image to speed up FFT calculations for non-standard image sizes print "checking prime factor" if primefactor.isGoodStack(dimx) is False: goodsize = primefactor.getNextEvenPrime(dimx) factor = float(goodsize) / float(dimx) apDisplay.printMsg("padding image: %ix%i to %ix%i" % (dimx,dimy,dimx*factor,dimy*factor)) mrcarray = apImage.mrcToArray(ace2inputpath,msg=False) # paddedmrc = imagefun.pad(mrcarray, None, factor) paddedmrc = apImage.frame_constant(mrcarray, (dimx*factor,dimy*factor), cval=mrcarray.mean()) ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc") apImage.arrayToMrc(paddedmrc,ace2inputpath,msg=False) inputparams = { 'input': ace2inputpath, 'cs': self.params['cs'], 'kv': imgdata['scope']['high tension']/1000.0, 'apix': apix, 'binby': self.params['bin'], } ### make standard input for ACE 2 apDisplay.printMsg("Ace2 executable: "+self.ace2exe) commandline = ( self.ace2exe + " -i " + str(inputparams['input']) + " -b " + str(inputparams['binby']) + " -c " + str(inputparams['cs']) + " -k " + str(inputparams['kv']) + " -a " + str(inputparams['apix']) + " -e " + str(self.params['edge_b'])+","+str(self.params['edge_t']) + " -r " + str(self.params['rotblur']) + "\n" ) ### run ace2 apDisplay.printMsg("running ace2 at "+time.asctime()) apDisplay.printColor(commandline, "purple") t0 = time.time() if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() ### check if ace2 worked basename = os.path.basename(ace2inputpath) imagelog = basename+".ctf.txt" if not os.path.isfile(imagelog) and self.stats['count'] <= 1: ### ace2 always crashes on first image??? .fft_wisdom file?? time.sleep(1) if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() if self.params['verbose'] is False: aceoutf.close() aceerrf.close() if not os.path.isfile(imagelog): lddcmd = "ldd "+self.ace2exe lddproc = subprocess.Popen(lddcmd, shell=True) lddproc.wait() apDisplay.printError("ace2 did not run") apDisplay.printMsg("ace2 completed in " + apDisplay.timeString(time.time()-t0)) ### parse log file self.ctfvalues = { 'cs': self.params['cs'], 'volts': imgdata['scope']['high tension'], } logf = open(imagelog, "r") apDisplay.printMsg("reading log file %s"%(imagelog)) for line in logf: sline = line.strip() if re.search("^Final Defocus: ", sline): ### old ACE2 apDisplay.printError("This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now") #parts = sline.split() #self.ctfvalues['defocus1'] = float(parts[2]) #self.ctfvalues['defocus2'] = float(parts[3]) ### convert to degrees #self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4])) elif re.search("^Final Defocus \(m,m,deg\):", sline): ### new ACE2 apDisplay.printMsg("Reading new ACE2 defocus") parts = sline.split() #print parts self.ctfvalues['defocus1'] = float(parts[3]) self.ctfvalues['defocus2'] = float(parts[4]) # ace2 defines negative angle from +x toward +y self.ctfvalues['angle_astigmatism'] = -float(parts[5]) elif re.search("^Amplitude Contrast:",sline): parts = sline.split() self.ctfvalues['amplitude_contrast'] = float(parts[2]) elif re.search("^Confidence:",sline): parts = sline.split() self.ctfvalues['confidence'] = float(parts[1]) self.ctfvalues['confidence_d'] = float(parts[1]) logf.close() ### summary stats apDisplay.printMsg("============") avgdf = (self.ctfvalues['defocus1']+self.ctfvalues['defocus2'])/2.0 ampconst = 100.0*self.ctfvalues['amplitude_contrast'] pererror = 100.0 * (self.ctfvalues['defocus1']-self.ctfvalues['defocus2']) / avgdf apDisplay.printMsg("Defocus: %.3f x %.3f um (%.2f percent astigmatism)"% (self.ctfvalues['defocus1']*1.0e6, self.ctfvalues['defocus2']*1.0e6, pererror )) apDisplay.printMsg("Angle astigmatism: %.2f degrees"%(self.ctfvalues['angle_astigmatism'])) apDisplay.printMsg("Amplitude contrast: %.2f percent"%(ampconst)) apDisplay.printColor("Final confidence: %.3f"%(self.ctfvalues['confidence']),'cyan') ### double check that the values are reasonable if avgdf > self.params['maxdefocus'] or avgdf < self.params['mindefocus']: apDisplay.printWarning("bad defocus estimate, not committing values to database") self.badprocess = True if ampconst < 0.0 or ampconst > 80.0: apDisplay.printWarning("bad amplitude contrast, not committing values to database") self.badprocess = True if self.ctfvalues['confidence'] < 0.2: apDisplay.printWarning("bad confidence value, not committing values to database") self.badprocess = True ## create power spectra jpeg mrcfile = imgdata['filename']+".mrc.edge.mrc" if os.path.isfile(mrcfile): jpegfile = os.path.join(self.powerspecdir, apDisplay.short(imgdata['filename'])+".jpg") ps = apImage.mrcToArray(mrcfile,msg=False) c = numpy.array(ps.shape)/2.0 ps[c[0]-0,c[1]-0] = ps.mean() ps[c[0]-1,c[1]-0] = ps.mean() ps[c[0]-0,c[1]-1] = ps.mean() ps[c[0]-1,c[1]-1] = ps.mean() #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) ps = numpy.log(ps+1.0) ps = (ps-ps.mean())/ps.std() cutoff = -2.0*ps.min() ps = numpy.where(ps > cutoff, cutoff, ps) cutoff = ps.mean() ps = numpy.where(ps < cutoff, cutoff, ps) #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) apImage.arrayToJpeg(ps, jpegfile, msg=False) apFile.removeFile(mrcfile) self.ctfvalues['graph3'] = jpegfile otherfiles = glob.glob(imgdata['filename']+".*.txt") ### remove extra debugging files for filename in otherfiles: if filename[-9:] == ".norm.txt": continue elif filename[-8:] == ".ctf.txt": continue else: apFile.removeFile(filename) if maskhighpass and os.path.isfile(ace2inputpath): apFile.removeFile(ace2inputpath) return
def real_fft3d(volume, *args, **kwargs): padshape = numpy.asarray(volume.shape)*1 padvolume = apImage.frame_constant(volume, padshape, volume.mean()) fft = fftpack.fftn(padvolume, *args, **kwargs) return fft
def real_fft2d(image, *args, **kwargs): padshape = numpy.asarray(image.shape)*1 padimage = apImage.frame_constant(image, padshape, image.mean()) fft = fftpack.fft2(padimage, *args, **kwargs) #normfft = (fft - fft.mean())/fft.std() return fft
def real_fft2d(self, image, *args, **kwargs): padshape = numpy.asarray(image.shape)*1 padimage = apImage.frame_constant(image, padshape, image.mean()) fft = fftpack.fft2(padimage, *args, **kwargs) return fft
def getTiltedRotateShift(img1, img2, tiltdiff, angle=0, bin=1, msg=True): """ takes two images tilted with respect to one another and tries to find overlap img1 (as numpy array) img2 (as numpy array) tiltdiff (in degrees) negative, img1 is more compressed (tilted) positive, img2 is more compressed (tilted) """ ### untilt images by stretching and compressing # choose angle s/t compressFactor = 1/stretchFactor # this only works if one image is untilted (RCT) of both images are opposite tilt (OTR) #halftilt = abs(tiltdiff)/2.0 halftiltrad = math.acos(math.sqrt(math.cos(abs(tiltdiff)/180.0*math.pi))) # go from zero tilt to half tilt compressFactor = math.cos(halftiltrad) # go from max tilt to half tilt stretchFactor = math.cos(halftiltrad) / math.cos(abs(tiltdiff)/180.0*math.pi) if tiltdiff > 0: if msg is True: apDisplay.printMsg("compress image 1") untilt1 = transformImage(img1, compressFactor, angle) untilt2 = transformImage(img2, stretchFactor, angle) xfactor = compressFactor else: if msg is True: apDisplay.printMsg("stretch image 1") untilt1 = transformImage(img1, stretchFactor, angle) untilt2 = transformImage(img2, compressFactor, angle) xfactor = stretchFactor ### filtering was done earlier filt1 = untilt1 filt2 = untilt2 if filt1.shape != filt2.shape: newshape = ( max(filt1.shape[0],filt2.shape[0]), max(filt1.shape[1],filt2.shape[1]) ) apDisplay.printMsg("Resizing images to: "+str(newshape)) filt1 = apImage.frame_constant(filt1, newshape, filt1.mean()) filt2 = apImage.frame_constant(filt2, newshape, filt2.mean()) ### cross-correlate cc = correlator.cross_correlate(filt1, filt2, pad=True) rad = min(cc.shape)/20.0 cc = apImage.highPassFilter(cc, radius=rad) cc = apImage.normRange(cc) cc = blackEdges(cc) cc = apImage.normRange(cc) cc = blackEdges(cc) cc = apImage.normRange(cc) cc = apImage.lowPassFilter(cc, radius=10.0) #find peak peakdict = peakfinder.findSubpixelPeak(cc, lpf=0) #import pprint #pprint.pprint(peak) pixpeak = peakdict['subpixel peak'] if msg is True: apDisplay.printMsg("Pixel peak: "+str(pixpeak)) apImage.arrayToJpegPlusPeak(cc, "guess-cross-ang"+str(abs(angle))+".jpg", pixpeak) rawpeak = numpy.array([pixpeak[1], pixpeak[0]]) #swap coord shift = numpy.asarray(correlator.wrap_coord(rawpeak, cc.shape))*bin if msg is True: apDisplay.printMsg("Found xy-shift btw two images" +";\n\t SNR= "+str(round(peakdict['snr'],2)) +";\n\t halftilt= "+str(round(halftiltrad*180/math.pi, 3)) +";\n\t compressFactor= "+str(round(compressFactor, 3)) +";\n\t stretchFactor= "+str(round(stretchFactor, 3)) +";\n\t xFactor= "+str(round(xfactor, 3)) +";\n\t rawpeak= "+str(numpy.around(rawpeak*bin, 1)) +";\n\t shift= "+str(numpy.around(shift, 1)) ) return shift, xfactor, peakdict['snr']
def processImage(self, imgdata): self.ctfvalues = {} bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True) apix = apDatabase.getPixelSize(imgdata) if (not (self.params['onepass'] and self.params['zeropass'])): maskhighpass = False ace2inputpath = os.path.join(imgdata['session']['image path'], imgdata['filename'] + ".mrc") else: maskhighpass = True filterimg = apImage.maskHighPassFilter(imgdata['image'], apix, 1, self.params['zeropass'], self.params['onepass']) ace2inputpath = os.path.join(self.params['rundir'], imgdata['filename'] + ".mrc") mrc.write(filterimg, ace2inputpath) # make sure that the image is a square dimx = imgdata['camera']['dimension']['x'] dimy = imgdata['camera']['dimension']['y'] if dimx != dimy: dims = [dimx, dimy] dims.sort() apDisplay.printMsg("resizing image: %ix%i to %ix%i" % (dimx, dimy, dims[0], dims[0])) mrcarray = apImage.mrcToArray(ace2inputpath, msg=False) clippedmrc = apImage.frame_cut(mrcarray, [dims[0], dims[0]]) ace2inputpath = os.path.join(self.params['rundir'], imgdata['filename'] + ".mrc") apImage.arrayToMrc(clippedmrc, ace2inputpath, msg=False) ### pad out image to speed up FFT calculations for non-standard image sizes print "checking prime factor" if primefactor.isGoodStack(dimx) is False: goodsize = primefactor.getNextEvenPrime(dimx) factor = float(goodsize) / float(dimx) apDisplay.printMsg("padding image: %ix%i to %ix%i" % (dimx, dimy, dimx * factor, dimy * factor)) mrcarray = apImage.mrcToArray(ace2inputpath, msg=False) # paddedmrc = imagefun.pad(mrcarray, None, factor) paddedmrc = apImage.frame_constant(mrcarray, (dimx * factor, dimy * factor), cval=mrcarray.mean()) ace2inputpath = os.path.join(self.params['rundir'], imgdata['filename'] + ".mrc") apImage.arrayToMrc(paddedmrc, ace2inputpath, msg=False) inputparams = { 'input': ace2inputpath, 'cs': self.params['cs'], 'kv': imgdata['scope']['high tension'] / 1000.0, 'apix': apix, 'binby': self.params['bin'], } ### make standard input for ACE 2 apDisplay.printMsg("Ace2 executable: " + self.ace2exe) commandline = (self.ace2exe + " -i " + str(inputparams['input']) + " -b " + str(inputparams['binby']) + " -c " + str(inputparams['cs']) + " -k " + str(inputparams['kv']) + " -a " + str(inputparams['apix']) + " -e " + str(self.params['edge_b']) + "," + str(self.params['edge_t']) + " -r " + str(self.params['rotblur']) + "\n") ### run ace2 apDisplay.printMsg("running ace2 at " + time.asctime()) apDisplay.printColor(commandline, "purple") t0 = time.time() if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() ### check if ace2 worked basename = os.path.basename(ace2inputpath) imagelog = basename + ".ctf.txt" if not os.path.isfile(imagelog) and self.stats['count'] <= 1: ### ace2 always crashes on first image??? .fft_wisdom file?? time.sleep(1) if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() if self.params['verbose'] is False: aceoutf.close() aceerrf.close() if not os.path.isfile(imagelog): lddcmd = "ldd " + self.ace2exe lddproc = subprocess.Popen(lddcmd, shell=True) lddproc.wait() apDisplay.printError("ace2 did not run") apDisplay.printMsg("ace2 completed in " + apDisplay.timeString(time.time() - t0)) ### parse log file self.ctfvalues = { 'cs': self.params['cs'], 'volts': imgdata['scope']['high tension'], } logf = open(imagelog, "r") apDisplay.printMsg("reading log file %s" % (imagelog)) for line in logf: sline = line.strip() if re.search("^Final Defocus: ", sline): ### old ACE2 apDisplay.printError( "This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now" ) #parts = sline.split() #self.ctfvalues['defocus1'] = float(parts[2]) #self.ctfvalues['defocus2'] = float(parts[3]) ### convert to degrees #self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4])) elif re.search("^Final Defocus \(m,m,deg\):", sline): ### new ACE2 apDisplay.printMsg("Reading new ACE2 defocus") parts = sline.split() #print parts self.ctfvalues['defocus1'] = float(parts[3]) self.ctfvalues['defocus2'] = float(parts[4]) # ace2 defines negative angle from +x toward +y self.ctfvalues['angle_astigmatism'] = -float(parts[5]) elif re.search("^Amplitude Contrast:", sline): parts = sline.split() self.ctfvalues['amplitude_contrast'] = float(parts[2]) elif re.search("^Confidence:", sline): parts = sline.split() self.ctfvalues['confidence'] = float(parts[1]) self.ctfvalues['confidence_d'] = float(parts[1]) logf.close() ### summary stats apDisplay.printMsg("============") avgdf = (self.ctfvalues['defocus1'] + self.ctfvalues['defocus2']) / 2.0 ampconst = 100.0 * self.ctfvalues['amplitude_contrast'] pererror = 100.0 * (self.ctfvalues['defocus1'] - self.ctfvalues['defocus2']) / avgdf apDisplay.printMsg( "Defocus: %.3f x %.3f um (%.2f percent astigmatism)" % (self.ctfvalues['defocus1'] * 1.0e6, self.ctfvalues['defocus2'] * 1.0e6, pererror)) apDisplay.printMsg("Angle astigmatism: %.2f degrees" % (self.ctfvalues['angle_astigmatism'])) apDisplay.printMsg("Amplitude contrast: %.2f percent" % (ampconst)) apDisplay.printColor( "Final confidence: %.3f" % (self.ctfvalues['confidence']), 'cyan') ### double check that the values are reasonable if avgdf > self.params['maxdefocus'] or avgdf < self.params[ 'mindefocus']: apDisplay.printWarning( "bad defocus estimate, not committing values to database") self.badprocess = True if ampconst < 0.0 or ampconst > 80.0: apDisplay.printWarning( "bad amplitude contrast, not committing values to database") self.badprocess = True if self.ctfvalues['confidence'] < 0.2: apDisplay.printWarning( "bad confidence value, not committing values to database") self.badprocess = True ## create power spectra jpeg mrcfile = imgdata['filename'] + ".mrc.edge.mrc" if os.path.isfile(mrcfile): jpegfile = os.path.join( self.powerspecdir, apDisplay.short(imgdata['filename']) + ".jpg") ps = apImage.mrcToArray(mrcfile, msg=False) c = numpy.array(ps.shape) / 2.0 ps[c[0] - 0, c[1] - 0] = ps.mean() ps[c[0] - 1, c[1] - 0] = ps.mean() ps[c[0] - 0, c[1] - 1] = ps.mean() ps[c[0] - 1, c[1] - 1] = ps.mean() #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) ps = numpy.log(ps + 1.0) ps = (ps - ps.mean()) / ps.std() cutoff = -2.0 * ps.min() ps = numpy.where(ps > cutoff, cutoff, ps) cutoff = ps.mean() ps = numpy.where(ps < cutoff, cutoff, ps) #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) apImage.arrayToJpeg(ps, jpegfile, msg=False) apFile.removeFile(mrcfile) self.ctfvalues['graph3'] = jpegfile otherfiles = glob.glob(imgdata['filename'] + ".*.txt") ### remove extra debugging files for filename in otherfiles: if filename[-9:] == ".norm.txt": continue elif filename[-8:] == ".ctf.txt": continue else: apFile.removeFile(filename) if maskhighpass and os.path.isfile(ace2inputpath): apFile.removeFile(ace2inputpath) return