def loopProcessImage(self, imgdata): """ Over-writes the particleLoop processImage and uses the appionLoop processImage """ #GET THE TILT PAIR tiltdata = apTiltPair.getTiltPair(imgdata) if tiltdata is None: return ### process images procimgpath = os.path.join(self.params['rundir'], imgdata['filename'] + '.dwn.mrc') if not os.path.isfile(procimgpath): apDisplay.printMsg("processing mrc for first image") apFindEM.processAndSaveImage(imgdata, params=self.params) procimg1 = apImage.mrcToArray(procimgpath, msg=False) procimgpath = os.path.join(self.params['rundir'], tiltdata['filename'] + '.dwn.mrc') if not os.path.isfile(procimgpath): apDisplay.printMsg("processing mrc for second image") apFindEM.processAndSaveImage(tiltdata, params=self.params) procimg2 = apImage.mrcToArray(procimgpath, msg=False) #RUN THE ALIGNER GUI result = self.runTiltAligner(imgdata, tiltdata) if self.badprocess is True: return numpeaks = len(self.peaktree1) #apDisplay.printMsg("Found "+str(numpeaks)+" particles for "+apDisplay.shortenImageName(imgdata['filename'])) self.stats['lastpeaks'] = numpeaks ### mask the images procimg1, procimg2 = apTiltTransform.maskOverlapRegion( procimg1, procimg2, self.data) #CREATE PEAK JPEG if self.threadJpeg is True: threading.Thread(target=apPeaks.createTiltedPeakJpeg, args=(imgdata, tiltdata, self.peaktree1, self.peaktree2, self.params, procimg1, procimg2)).start() else: apPeaks.createTiltedPeakJpeg(imgdata, tiltdata, self.peaktree1, self.peaktree2, self.params, procimg1, procimg2) #EXTRA DONE DICT CALL self._writeDoneDict(tiltdata['filename'])
def runTestShift(img1name, img2name, imgpath, tiltdiff, coord): img1path = os.path.join(imgpath, img1name) img2path = os.path.join(imgpath, img2name) print img1path print img2path img1 = apImage.binImg(apImage.mrcToArray(img1path),4) img2 = apImage.binImg(apImage.mrcToArray(img2path),4) apImage.arrayToMrc(img1,"img1a-raw.mrc") apImage.arrayToMrc(img2,"img2a-raw.mrc") origin, newpart, snr = apTiltShift.getTiltedCoordinates(img1, img2, tiltdiff, coord) apImage.arrayToJpegPlusPeak(img1, "img1a-guess.jpg", (origin[1], origin[0])) apImage.arrayToJpegPlusPeak(img2, "img2a-guess.jpg", (newpart[1], newpart[0]))
def loopProcessImage(self, imgdata): """ Over-writes the particleLoop processImage and uses the appionLoop processImage """ #GET THE TILT PAIR tiltdata = apTiltPair.getTiltPair(imgdata) if tiltdata is None: return ### process images procimgpath = os.path.join(self.params['rundir'], imgdata['filename']+'.dwn.mrc') if not os.path.isfile(procimgpath): apDisplay.printMsg("processing mrc for first image") apFindEM.processAndSaveImage(imgdata, params=self.params) procimg1 = apImage.mrcToArray(procimgpath, msg=False) procimgpath = os.path.join(self.params['rundir'], tiltdata['filename']+'.dwn.mrc') if not os.path.isfile(procimgpath): apDisplay.printMsg("processing mrc for second image") apFindEM.processAndSaveImage(tiltdata, params=self.params) procimg2 = apImage.mrcToArray(procimgpath, msg=False) #RUN THE ALIGNER GUI result = self.runTiltAligner(imgdata, tiltdata) if self.badprocess is True: return numpeaks = len(self.peaktree1) #apDisplay.printMsg("Found "+str(numpeaks)+" particles for "+apDisplay.shortenImageName(imgdata['filename'])) self.stats['lastpeaks'] = numpeaks ### mask the images procimg1, procimg2 = apTiltTransform.maskOverlapRegion(procimg1, procimg2, self.data) #CREATE PEAK JPEG if self.threadJpeg is True: threading.Thread(target=apPeaks.createTiltedPeakJpeg, args=(imgdata, tiltdata, self.peaktree1, self.peaktree2, self.params, procimg1, procimg2)).start() else: apPeaks.createTiltedPeakJpeg(imgdata, tiltdata, self.peaktree1, self.peaktree2, self.params, procimg1, procimg2) #EXTRA DONE DICT CALL self._writeDoneDict(tiltdata['filename'])
def loopProcessImage(self, imgdata): """ setup like this to override things """ self.filtimgpath = os.path.join(self.params['rundir'], imgdata['filename']+'.dwn.mrc') if os.path.isfile(self.filtimgpath) and self.params['continue'] is True: apDisplay.printMsg("reading filtered image from mrc file") self.filtarray = apImage.mrcToArray(self.filtimgpath, msg=False) peaktree = self.processImage(imgdata, self.filtarray) return peaktree
def openImageFile(self, filename): self.filename = filename if filename[-4:] == '.spi': array = apImage.spiderToArray(filename, msg=False) return array elif filename[-4:] == '.mrc': array = apImage.mrcToArray(filename, msg=False) return array else: image = Image.open(filename) array = apImage.imageToArray(image, msg=False) array = array.astype(numpy.float32) return array return None
def loopProcessImage(self, imgdata): """ setup like this to override things """ self.filtimgpath = os.path.join(self.params['rundir'], imgdata['filename'] + '.dwn.mrc') if os.path.isfile( self.filtimgpath) and self.params['continue'] is True: apDisplay.printMsg("reading filtered image from mrc file") self.filtarray = apImage.mrcToArray(self.filtimgpath, msg=False) peaktree = self.processImage(imgdata, self.filtarray) return peaktree
def getTemplates(params): """ Inputs: params['templateIds'], a list of template ids params['apix'], desired pixel size params['rundir'], output directory image processing params Processing: Copies, scales, and filters templates Outputs: params['templatelist'], a list of template file basenames """ apDisplay.printMsg("getting templates") if not params['templateIds']: apDisplay.printError("No template ids were specified") params['templatelist'] = [] #list of scaled files for i,templateid in enumerate(params['templateIds']): index = i+1 #print templateid templateid = int(templateid) if templateid < 0: continue #QUERY DB FOR TEMPLATE INFO templatedata = appiondata.ApTemplateImageData.direct_query(abs(templateid)) if not (templatedata): apDisplay.printError("Template Id "+str(templateid)+" was not found in database.") #COPY THE FILE OVER origtemplatepath = os.path.join(templatedata['path']['path'], templatedata['templatename']) if not os.path.isfile(origtemplatepath): apDisplay.printError("Template file not found: "+origtemplatepath) apDisplay.printMsg("getting template: "+origtemplatepath) copytemplatepath = os.path.join(params['rundir'], "origTemplate"+str(index)+".mrc") scaletemplatepath = os.path.join(params['rundir'], "scaledTemplate"+str(index)+".mrc") filtertemplatepath = os.path.join(params['rundir'], "filterTemplate"+str(index)+".mrc") #masktemplatepath = os.path.join(params['rundir'], "maskTemplate"+str(index)+".mrc") shutil.copyfile(origtemplatepath, copytemplatepath) #RESCALE THE TEMPLATE templatearray = apImage.mrcToArray(copytemplatepath) #scale to correct apix scalefactor = templatedata['apix'] / params['apix'] if abs(scalefactor - 1.0) > 0.01: apDisplay.printMsg("rescaling template "+str(index)+": "+str(templatedata['apix'])+"->"+str(params['apix'])) templatearray = scaleTemplate(templatearray, scalefactor) apImage.arrayToMrc(templatearray, scaletemplatepath, msg=False) #bin and filter templatearray = apImage.preProcessImage(templatearray, params=params, highpass=0, planeReg=False, invert=False) #write to file apImage.arrayToMrc(templatearray, filtertemplatepath, msg=False) ### MASK THE TEMPLATE AND SAVE #mask the template, visual purposes only #maskrad = params['diam']/params['apix']/params['bin']/2.0 #maskarray = #apImage.arrayToMrc(templatearray, masktemplatepath, msg=False) #ADD TO TEMPLATE LIST params['templatelist'].append(os.path.basename(filtertemplatepath)) ### ADD MIRROR IF REQUESTED if 'templatemirrors' in params and params['templatemirrors'] is True: mirrortemplatepath = os.path.join(params['rundir'], "mirrorTemplate"+str(index)+".mrc") mirrorarray = numpy.fliplr(templatearray) apImage.arrayToMrc(mirrorarray, mirrortemplatepath, msg=False) params['templatelist'].append(os.path.basename(mirrortemplatepath)) #FINISH LOOP OVER template ids #Set the apix params['templateapix'] = params['apix'] apDisplay.printMsg("scaled & filtered "+str(len(params['templatelist']))+" file(s)") return params['templatelist']
def processImage(self, imgdata): self.ctfvalues = {} bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True) apix = apDatabase.getPixelSize(imgdata) if (not (self.params['onepass'] and self.params['zeropass'])): maskhighpass = False ace2inputpath = os.path.join(imgdata['session']['image path'],imgdata['filename']+".mrc") else: maskhighpass = True filterimg = apImage.maskHighPassFilter(imgdata['image'],apix,1,self.params['zeropass'],self.params['onepass']) ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc") mrc.write(filterimg,ace2inputpath) # make sure that the image is a square dimx = imgdata['camera']['dimension']['x'] dimy = imgdata['camera']['dimension']['y'] if dimx != dimy: dims = [dimx,dimy] dims.sort() apDisplay.printMsg("resizing image: %ix%i to %ix%i" % (dimx,dimy,dims[0],dims[0])) mrcarray = apImage.mrcToArray(ace2inputpath,msg=False) clippedmrc = apImage.frame_cut(mrcarray,[dims[0],dims[0]]) ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc") apImage.arrayToMrc(clippedmrc,ace2inputpath,msg=False) ### pad out image to speed up FFT calculations for non-standard image sizes print "checking prime factor" if primefactor.isGoodStack(dimx) is False: goodsize = primefactor.getNextEvenPrime(dimx) factor = float(goodsize) / float(dimx) apDisplay.printMsg("padding image: %ix%i to %ix%i" % (dimx,dimy,dimx*factor,dimy*factor)) mrcarray = apImage.mrcToArray(ace2inputpath,msg=False) # paddedmrc = imagefun.pad(mrcarray, None, factor) paddedmrc = apImage.frame_constant(mrcarray, (dimx*factor,dimy*factor), cval=mrcarray.mean()) ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc") apImage.arrayToMrc(paddedmrc,ace2inputpath,msg=False) inputparams = { 'input': ace2inputpath, 'cs': self.params['cs'], 'kv': imgdata['scope']['high tension']/1000.0, 'apix': apix, 'binby': self.params['bin'], } ### make standard input for ACE 2 apDisplay.printMsg("Ace2 executable: "+self.ace2exe) commandline = ( self.ace2exe + " -i " + str(inputparams['input']) + " -b " + str(inputparams['binby']) + " -c " + str(inputparams['cs']) + " -k " + str(inputparams['kv']) + " -a " + str(inputparams['apix']) + " -e " + str(self.params['edge_b'])+","+str(self.params['edge_t']) + " -r " + str(self.params['rotblur']) + "\n" ) ### run ace2 apDisplay.printMsg("running ace2 at "+time.asctime()) apDisplay.printColor(commandline, "purple") t0 = time.time() if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() ### check if ace2 worked basename = os.path.basename(ace2inputpath) imagelog = basename+".ctf.txt" if not os.path.isfile(imagelog) and self.stats['count'] <= 1: ### ace2 always crashes on first image??? .fft_wisdom file?? time.sleep(1) if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() if self.params['verbose'] is False: aceoutf.close() aceerrf.close() if not os.path.isfile(imagelog): lddcmd = "ldd "+self.ace2exe lddproc = subprocess.Popen(lddcmd, shell=True) lddproc.wait() apDisplay.printError("ace2 did not run") apDisplay.printMsg("ace2 completed in " + apDisplay.timeString(time.time()-t0)) ### parse log file self.ctfvalues = { 'cs': self.params['cs'], 'volts': imgdata['scope']['high tension'], } logf = open(imagelog, "r") apDisplay.printMsg("reading log file %s"%(imagelog)) for line in logf: sline = line.strip() if re.search("^Final Defocus: ", sline): ### old ACE2 apDisplay.printError("This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now") #parts = sline.split() #self.ctfvalues['defocus1'] = float(parts[2]) #self.ctfvalues['defocus2'] = float(parts[3]) ### convert to degrees #self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4])) elif re.search("^Final Defocus \(m,m,deg\):", sline): ### new ACE2 apDisplay.printMsg("Reading new ACE2 defocus") parts = sline.split() #print parts self.ctfvalues['defocus1'] = float(parts[3]) self.ctfvalues['defocus2'] = float(parts[4]) # ace2 defines negative angle from +x toward +y self.ctfvalues['angle_astigmatism'] = -float(parts[5]) elif re.search("^Amplitude Contrast:",sline): parts = sline.split() self.ctfvalues['amplitude_contrast'] = float(parts[2]) elif re.search("^Confidence:",sline): parts = sline.split() self.ctfvalues['confidence'] = float(parts[1]) self.ctfvalues['confidence_d'] = float(parts[1]) logf.close() ### summary stats apDisplay.printMsg("============") avgdf = (self.ctfvalues['defocus1']+self.ctfvalues['defocus2'])/2.0 ampconst = 100.0*self.ctfvalues['amplitude_contrast'] pererror = 100.0 * (self.ctfvalues['defocus1']-self.ctfvalues['defocus2']) / avgdf apDisplay.printMsg("Defocus: %.3f x %.3f um (%.2f percent astigmatism)"% (self.ctfvalues['defocus1']*1.0e6, self.ctfvalues['defocus2']*1.0e6, pererror )) apDisplay.printMsg("Angle astigmatism: %.2f degrees"%(self.ctfvalues['angle_astigmatism'])) apDisplay.printMsg("Amplitude contrast: %.2f percent"%(ampconst)) apDisplay.printColor("Final confidence: %.3f"%(self.ctfvalues['confidence']),'cyan') ### double check that the values are reasonable if avgdf > self.params['maxdefocus'] or avgdf < self.params['mindefocus']: apDisplay.printWarning("bad defocus estimate, not committing values to database") self.badprocess = True if ampconst < 0.0 or ampconst > 80.0: apDisplay.printWarning("bad amplitude contrast, not committing values to database") self.badprocess = True if self.ctfvalues['confidence'] < 0.2: apDisplay.printWarning("bad confidence value, not committing values to database") self.badprocess = True ## create power spectra jpeg mrcfile = imgdata['filename']+".mrc.edge.mrc" if os.path.isfile(mrcfile): jpegfile = os.path.join(self.powerspecdir, apDisplay.short(imgdata['filename'])+".jpg") ps = apImage.mrcToArray(mrcfile,msg=False) c = numpy.array(ps.shape)/2.0 ps[c[0]-0,c[1]-0] = ps.mean() ps[c[0]-1,c[1]-0] = ps.mean() ps[c[0]-0,c[1]-1] = ps.mean() ps[c[0]-1,c[1]-1] = ps.mean() #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) ps = numpy.log(ps+1.0) ps = (ps-ps.mean())/ps.std() cutoff = -2.0*ps.min() ps = numpy.where(ps > cutoff, cutoff, ps) cutoff = ps.mean() ps = numpy.where(ps < cutoff, cutoff, ps) #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) apImage.arrayToJpeg(ps, jpegfile, msg=False) apFile.removeFile(mrcfile) self.ctfvalues['graph3'] = jpegfile otherfiles = glob.glob(imgdata['filename']+".*.txt") ### remove extra debugging files for filename in otherfiles: if filename[-9:] == ".norm.txt": continue elif filename[-8:] == ".ctf.txt": continue else: apFile.removeFile(filename) if maskhighpass and os.path.isfile(ace2inputpath): apFile.removeFile(ace2inputpath) return
def fakeOutput(imgname, ccmapfile, params): a = apImage.mrcToArray(imgname) a = numpy.zeros(a.shape) apImage.arrayToMrc(a, ccmapfile) apFile.safeCopy('/home/acheng/Projects/Gfindem/example.box', getBoxFileName(ccmapfile))
def fakeOutput(imgname,ccmapfile,params): a = apImage.mrcToArray(imgname) a = numpy.zeros(a.shape) apImage.arrayToMrc(a,ccmapfile) apFile.safeCopy('/home/acheng/Projects/Gfindem/example.box', getBoxFileName(ccmapfile))
def runSpectralFindEM(imgdict, params, thread=False): """ runs a separate thread of findem.exe for each template to get cross-correlation maps """ imgname = imgdict['filename'] dwnimgname = os.path.splitext(imgname)[0] + ".dwn.mrc" os.chdir(params['rundir']) joblist = [] ccmaplist = [] processAndSaveImage(imgdict, params) ### FindEM crashes when an input image is longer than 76 characters if len(dwnimgname) > 76: randlink = ''.join( random.choice(string.ascii_uppercase + string.digits) for x in range(10)) randlink += '.mrc' os.symlink(dwnimgname, randlink) if len(params['templatelist']) < 1: apDisplay.printError("templatelist == 0; there are no templates") for i, templatename in enumerate(params['templatelist']): classavg = i + 1 #First round: normal findem: template x image numstr = "%03d" % (100 + classavg) ccmapfile1 = "cccmaxmap" + numstr + ".mrc" apFile.removeFile(ccmapfile1) params["startang" + str(100 + classavg)] = params["startang" + str(classavg)] params["endang" + str(100 + classavg)] = params["endang" + str(classavg)] params["incrang" + str(100 + classavg)] = params["incrang" + str(classavg)] if len(dwnimgname) > 76: feed = findEMString(100 + classavg, templatename, randlink, ccmapfile1, params) else: feed = findEMString(100 + classavg, templatename, dwnimgname, ccmapfile1, params) execFindEM(feed) #Second round: template x template numstr = "%03d" % (200 + classavg) ccmapfile2 = "cccmaxmap" + numstr + ".mrc" apFile.removeFile(ccmapfile2) params["startang" + str(200 + classavg)] = params["startang" + str(classavg)] params["endang" + str(200 + classavg)] = params["endang" + str(classavg)] params["incrang" + str(200 + classavg)] = params["incrang" + str(classavg)] feed = findEMString(200 + classavg, templatename, templatename, ccmapfile2, params) execFindEM(feed) #Final round: (template x template) x (template x image) = spectral numstr = "%03d" % (300 + classavg) ccmapfile3 = "cccmaxmap" + numstr + ".mrc" apFile.removeFile(ccmapfile3) params["startang" + str(300 + classavg)] = 0 params["endang" + str(300 + classavg)] = 10 params["incrang" + str(300 + classavg)] = 20 feed = findEMString(300 + classavg, ccmapfile2, ccmapfile1, ccmapfile3, params) execFindEM(feed) #READ OUTPUT FILE if not os.path.isfile(ccmapfile3): apDisplay.printError("findem.exe did not run or crashed.\n" + "Did you source useappion.sh?") else: ccmaxmap = apImage.mrcToArray(ccmapfile3) ccmaplist.append(ccmaxmap) return ccmaplist
def processImage(self, imgdata): """ time ./ctffind3.exe << eof micrograph.mrc montage.pow 2.0, 200.0, 0.07, 60000, 7.0, 2 #! CS[mm], HT[kV], AmpCnst, XMAG, DStep[um], PAve 128, 200.0, 8.0, 5000.0, 40000.0, 5000.0 #! Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep eof CARD 1: Input file name for image CARD 2: Output file name to check result CARD 3: CS[mm], HT[kV], AmpCnst, XMAG, DStep[um],PAve CARD 4: Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep, dAst[A] CTFTILT also asks for TiltA[deg], TiltR[deg] at CARD4 The output image file to check the result of the fitting shows the filtered average power spectrum of the input image in one half, and the fitted CTF (squared) in the other half. The two halves should agree very well for a successful fit. CS: Spherical aberration coefficient of the objective in mm HT: Electron beam voltage in kV AmpCnst: Amount of amplitude contrast (fraction). For ice images 0.07, for negative stain about 0.15. XMAG: Magnification of original image DStep: Pixel size on scanner in microns, or apix*mag/10000 PAve: Pixel averaging (PAve x PAve) for input image Box: Tile size. The program divides the image into square tiles and calculates the average power spectrum. Tiles with a significantly higher or lower variance are excluded; these are parts of the image which are unlikely to contain useful information (beam edge, film number etc). IMPORTANT: Box must have a even pixel dimensions. ResMin: Low resolution end of data to be fitted. ResMaX: High resolution end of data to be fitted. dFMin: Starting defocus value for grid search in Angstrom. Positive values represent an underfocus. The program performs a systematic grid search of defocus values and astigmatism before fitting a CTF to machine precision. dFMax: End defocus value for grid search in Angstrom. FStep: Step width for grid search in Angstrom. dAst: An additional parameter, dAst, was added to CARD 4 to restrain the amount of astigmatism in the CTF fit. This makes the fitting procedure more robust, especially in cases where the Thon rings are not easily visible. TiltA: guessed tilt angle TiltR: angular range for initial coarse search """ #get Defocus in Angstroms self.ctfvalues = {} if self.params['nominal'] is not None: nominal = abs(self.params['nominal']*1e4) else: nominal = abs(imgdata['scope']['defocus']*-1.0e10) ctfvalue = ctfdb.getBestCtfByResolution(imgdata) if ctfvalue is not None: """ ## CTFFIND V3.5 (7-March-2012) prefers the smaller of the two values for astigmatic images I found that say you have an image with 1.1um and 1.5um defocus astigmatism. If you give CTFFIND the average value of 1.3um for the defocus and 0.4um astig (dast) then it will try to fit 1.3um and 1.8um, so you need to give it the minimum value (1.1um) for it to fit 1.1um and 1.5um. """ bestdef = min(ctfvalue['defocus1'],ctfvalue['defocus2'])*1.0e10 else: bestdef = nominal if ctfvalue is not None and self.params['bestdb'] is True: bestampcontrast = round(ctfvalue['amplitude_contrast'],3) beststigdiff = round(abs(ctfvalue['defocus1'] - ctfvalue['defocus2'])*1e10,1) else: bestampcontrast = self.params['amp'+self.params['medium']] beststigdiff = self.params['dast'] if ctfvalue is not None and self.params['bestdb'] is True: ### set res max from resolution_80_percent gmean = (ctfvalue['resolution_80_percent']*ctfvalue['resolution_50_percent']*self.params['resmax'])**(1/3.) if gmean < self.params['resmin']: # replace only if valid Issue #3291, #3547 self.params['resmax'] = round(gmean,2) apDisplay.printColor("Setting resmax to the geometric mean of resolution values", "purple") # dstep is the physical detector pixel size dstep = None if 'camera' in imgdata and imgdata['camera'] and imgdata['camera']['pixel size']: dstep = imgdata['camera']['pixel size']['x'] if dstep is None: dstep = apDatabase.getPixelSize(imgdata)*imgdata['scope']['magnification']/10000.0 dstep /=1e6 dstep = float(dstep) mpixelsize = apDatabase.getPixelSize(imgdata)*1e-10 if self.params['apix_man'] is not None: mpixelsize = self.params['apix_man']*1e-10 xmag = dstep / mpixelsize apDisplay.printMsg("Xmag=%d, dstep=%.2e, mpix=%.2e"%(xmag, dstep, mpixelsize)) inputparams = { 'orig': os.path.join(imgdata['session']['image path'], imgdata['filename']+".mrc"), 'input': apDisplay.short(imgdata['filename'])+".mrc", 'output': apDisplay.short(imgdata['filename'])+"-pow.mrc", 'cs': self.params['cs'], 'kv': imgdata['scope']['high tension']/1000.0, 'ampcnst': bestampcontrast, 'xmag': xmag, 'dstep': dstep*1e6, 'pixavg': self.params['bin'], 'box': self.params['fieldsize'], 'resmin': self.params['resmin'], 'resmax': self.params['resmax'], 'defstep': self.params['defstep'], #round(defocus/32.0, 1), 'dast': beststigdiff, } defrange = self.params['defstep'] * self.params['numstep'] ## do 25 steps in either direction inputparams['defmin']= round(bestdef-defrange, 1) #in meters if inputparams['defmin'] < 0: apDisplay.printWarning("Defocus minimum is less than zero") inputparams['defmin'] = inputparams['defstep'] inputparams['defmax']= round(bestdef+defrange, 1) #in meters apDisplay.printColor("Defocus search range: %d A to %d A (%.2f to %.2f um)" %(inputparams['defmin'], inputparams['defmax'], inputparams['defmin']*1e-4, inputparams['defmax']*1e-4), "cyan") ### secondary lock check right before it starts on the real part if self.params['parallel'] and os.path.isfile(apDisplay.short(imgdata['filename'])+".mrc"): # This is a secondary image lock check, checking the first output of the process. # It alone is not good enough apDisplay.printWarning('Some other parallel process is working on the same image. Skipping') return ### create local link to image if not os.path.exists(inputparams['input']): os.symlink(inputparams['orig'], inputparams['input']) ### make standard input for ctf estimation line1cmd = inputparams['input']+"\n" line2cmd = inputparams['output']+"\n" line3cmd = ( str(inputparams['cs'])+"," + str(inputparams['kv'])+"," + str(inputparams['ampcnst'])+"," + str(inputparams['xmag'])+"," + str(inputparams['dstep'])+"," + str(inputparams['pixavg'])+"\n") line4cmd = ( str(inputparams['box'])+"," + str(inputparams['resmin'])+"," + str(inputparams['resmax'])+"," + str(inputparams['defmin'])+"," + str(inputparams['defmax'])+"," + str(inputparams['defstep'])+"," + str(inputparams['dast'])) ### additional ctftilt parameters if self.params['ctftilt'] is True: tiltang = apDatabase.getTiltAngleDeg(imgdata) line4cmd += (","+str(tiltang)+",10") line4cmd += "\n" if os.path.isfile(inputparams['output']): # program crashes if this file exists apFile.removeFile(inputparams['output']) t0 = time.time() apDisplay.printMsg("running ctf estimation at "+time.asctime()) ctfproglog = os.path.join(self.logdir, os.path.splitext(imgdata['filename'])[0]+"-ctfprog.log") logf = open(ctfproglog, "w") ctfprogproc = subprocess.Popen(self.ctfprgmexe, shell=True, stdin=subprocess.PIPE, stdout=logf) apDisplay.printColor(self.ctfprgmexe, "magenta") apDisplay.printColor(line1cmd.strip(),"magenta") apDisplay.printColor(line2cmd.strip(),"magenta") apDisplay.printColor(line3cmd.strip(),"magenta") apDisplay.printColor(line4cmd.strip(),"magenta") ctfprogproc.stdin.write(line1cmd) ctfprogproc.stdin.write(line2cmd) ctfprogproc.stdin.write(line3cmd) ctfprogproc.stdin.write(line4cmd) ctfprogproc.communicate() logf.close() apDisplay.printMsg("ctf estimation completed in "+apDisplay.timeString(time.time()-t0)) #apFile.removeFile(inputparams['input']) ### parse ctf estimation output self.ctfvalues = {} logf = open(ctfproglog, "r") ## ctffind & ctftilt have diff # values numvals = 6 if self.params['ctftilt'] is True: numvals=8 for line in logf: sline = line.strip() if sline[-12:] == "Final Values": #print sline if '**********' in sline: sline = re.sub('**********', ' **********', sline) bits = sline.split() if len(bits) != numvals: apDisplay.printError("wrong number of values in "+str(bits)) for i,bit in enumerate(bits[0:(numvals-2)]): bits[i] = float(bit) self.ctfvalues = { 'defocus1': float(bits[0])*1e-10, 'defocus2': float(bits[1])*1e-10, # WARNING: this is the negative of the direct result 'angle_astigmatism': float(bits[2]), 'amplitude_contrast': inputparams['ampcnst'], 'cross_correlation': float(bits[numvals-3]), 'nominal': nominal*1e-10, 'defocusinit': bestdef*1e-10, 'cs': self.params['cs'], 'volts': imgdata['scope']['high tension'], 'confidence': float(bits[numvals-3]), 'confidence_d': round(math.sqrt(abs(float(bits[numvals-3]))), 5) } if self.params['ctftilt'] is True: self.ctfvalues['tilt_axis_angle']=float(bits[3]) self.ctfvalues['tilt_angle']=float(bits[4]) ### write to log file f = open("ctfvalues.log", "a") f.write("=== "+imgdata['filename']+" ===\n") if not self.ctfvalues: nominaldf = imgdata['scope']['defocus'] else: nominaldf = self.ctfvalues['nominal'] line1 = ("nominal=%.1e, bestdef=%.1e," % ( nominaldf, self.ctfvalues['defocusinit'])) if self.params['ctftilt'] is True: self.ctfvalues['origtiltang'] = tiltang line1+=" tilt=%.1f,"%tiltang apDisplay.printMsg(line1) f.write(line1) line2 = ("def_1=%.1e, def_2=%.1e, astig_angle=%.1f, cross_corr=%.3f,\n" % ( self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'], self.ctfvalues['cross_correlation'] )) if self.params['ctftilt'] is True: line2+= ("tilt_angle=%.1f, tilt_axis_angle=%.1f,\n" % (self.ctfvalues['tilt_angle'], self.ctfvalues['tilt_axis_angle'])) apDisplay.printMsg(line2) f.write(line2) f.close() #convert powerspectra to JPEG outputjpgbase = os.path.basename(os.path.splitext(inputparams['output'])[0]+".jpg") self.lastjpg = outputjpgbase outputjpg = os.path.join(self.powerspecdir, self.lastjpg) powspec = apImage.mrcToArray(inputparams['output']) apImage.arrayToJpeg(powspec, outputjpg) shutil.move(inputparams['output'], os.path.join(self.powerspecdir, inputparams['output'])) self.ctfvalues['graph1'] = outputjpg #apFile.removeFile(inputparams['input']) return
def getTemplates(params): """ Inputs: params['templateIds'], a list of template ids params['apix'], desired pixel size params['rundir'], output directory image processing params Processing: Copies, scales, and filters templates Outputs: params['templatelist'], a list of template file basenames """ apDisplay.printMsg("getting templates") if not params['templateIds']: apDisplay.printError("No template ids were specified") params['templatelist'] = [] #list of scaled files for i, templateid in enumerate(params['templateIds']): index = i + 1 #print templateid templateid = int(templateid) if templateid < 0: continue #QUERY DB FOR TEMPLATE INFO templatedata = appiondata.ApTemplateImageData.direct_query( abs(templateid)) if not (templatedata): apDisplay.printError("Template Id " + str(templateid) + " was not found in database.") #COPY THE FILE OVER origtemplatepath = os.path.join(templatedata['path']['path'], templatedata['templatename']) if not os.path.isfile(origtemplatepath): apDisplay.printError("Template file not found: " + origtemplatepath) apDisplay.printMsg("getting template: " + origtemplatepath) copytemplatepath = os.path.join(params['rundir'], "origTemplate" + str(index) + ".mrc") scaletemplatepath = os.path.join( params['rundir'], "scaledTemplate" + str(index) + ".mrc") filtertemplatepath = os.path.join( params['rundir'], "filterTemplate" + str(index) + ".mrc") #masktemplatepath = os.path.join(params['rundir'], "maskTemplate"+str(index)+".mrc") shutil.copyfile(origtemplatepath, copytemplatepath) #RESCALE THE TEMPLATE templatearray = apImage.mrcToArray(copytemplatepath) #scale to correct apix scalefactor = templatedata['apix'] / params['apix'] if abs(scalefactor - 1.0) > 0.01: apDisplay.printMsg("rescaling template " + str(index) + ": " + str(templatedata['apix']) + "->" + str(params['apix'])) templatearray = scaleTemplate(templatearray, scalefactor) apImage.arrayToMrc(templatearray, scaletemplatepath, msg=False) #bin and filter templatearray = apImage.preProcessImage(templatearray, params=params, highpass=0, planeReg=False, invert=False) #write to file apImage.arrayToMrc(templatearray, filtertemplatepath, msg=False) ### MASK THE TEMPLATE AND SAVE #mask the template, visual purposes only #maskrad = params['diam']/params['apix']/params['bin']/2.0 #maskarray = #apImage.arrayToMrc(templatearray, masktemplatepath, msg=False) #ADD TO TEMPLATE LIST params['templatelist'].append(os.path.basename(filtertemplatepath)) ### ADD MIRROR IF REQUESTED if 'templatemirrors' in params and params['templatemirrors'] is True: mirrortemplatepath = os.path.join( params['rundir'], "mirrorTemplate" + str(index) + ".mrc") mirrorarray = numpy.fliplr(templatearray) apImage.arrayToMrc(mirrorarray, mirrortemplatepath, msg=False) params['templatelist'].append(os.path.basename(mirrortemplatepath)) #FINISH LOOP OVER template ids #Set the apix params['templateapix'] = params['apix'] apDisplay.printMsg("scaled & filtered " + str(len(params['templatelist'])) + " file(s)") return params['templatelist']
def runFindEM(imgdict, params, thread=False): """ runs a separate thread of findem.exe for each template to get cross-correlation maps """ ### check image processAndSaveImage(imgdict, params) dwnimgname = imgdict['filename']+".dwn.mrc" if not os.path.isfile(dwnimgname): apDisplay.printError("cound not find image to process: "+dwnimgname) ### FindEM crashes when an input image is longer than 76 characters if len(dwnimgname) > 76: randlink = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(10)) randlink+= '.mrc' os.symlink(dwnimgname, randlink) ### check template if len(params['templatelist']) < 1: apDisplay.printError("templatelist == 0; there are no templates") joblist = [] ccmapfilelist = [] # For FindEM2 # ### generate circular mask for FindEM2 # apDisplay.printMsg("creating mask file for template matching") # img = apImage.mrcToArray(params['templatelist'][0]) # circlemask = 1 - imagefun.filled_circle(img.shape,img.shape[0]/2*0.95) # apImage.arrayToMrc(circlemask,"tmpmask.mrc") # del img,circlemask workimg = randlink if len(dwnimgname) > 76 else dwnimgname ### create list of inputs for findem threads feeds = [] for i,templatename in enumerate(params['templatelist']): classavg = i + 1 # OUTPUT FILE NAME numstr = "%03d" % classavg ccmapfile="cccmaxmap"+numstr+".mrc" apFile.removeFile(ccmapfile) feeds.append(findEMString(classavg, templatename, workimg, ccmapfile, params)) #STORE OUTPUT FILE ccmapfilelist.append(ccmapfile) ### launch findem threads t0 = time.time() findemexe = getFindEMPath() pool = multiprocessing.Pool(processes=params['nproc']) runner = findemrunner(findemexe,len(params['templatelist'])) for i,feed in enumerate(feeds): pool.apply_async(runner, (i,feed)) pool.close() pool.join() apDisplay.printMsg("\nFindEM finished in "+apDisplay.timeString(time.time()-t0)+"\n") # For FindEM2 # os.remove("tmpmask.mrc") ### READ OUTPUT FILES ccmaplist = [] for ccmapfile in ccmapfilelist: if not os.path.isfile(ccmapfile): apDisplay.printError("findem.exe did not run or crashed.\n") ccmaxmap = apImage.mrcToArray(ccmapfile) ccmaplist.append(ccmaxmap) return ccmaplist
def runSpectralFindEM(imgdict, params, thread=False): """ runs a separate thread of findem.exe for each template to get cross-correlation maps """ imgname = imgdict['filename'] dwnimgname = os.path.splitext(imgname)[0]+".dwn.mrc" os.chdir(params['rundir']) joblist = [] ccmaplist = [] processAndSaveImage(imgdict, params) ### FindEM crashes when an input image is longer than 76 characters if len(dwnimgname) > 76: randlink = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(10)) randlink+= '.mrc' os.symlink(dwnimgname, randlink) if len(params['templatelist']) < 1: apDisplay.printError("templatelist == 0; there are no templates") for i,templatename in enumerate(params['templatelist']): classavg = i + 1 #First round: normal findem: template x image numstr = "%03d" % (100+classavg) ccmapfile1 = "cccmaxmap"+numstr+".mrc" apFile.removeFile(ccmapfile1) params["startang"+str(100+classavg)] = params["startang"+str(classavg)] params["endang"+str(100+classavg)] = params["endang"+str(classavg)] params["incrang"+str(100+classavg)] = params["incrang"+str(classavg)] if len(dwnimgname) > 76: feed = findEMString(100+classavg, templatename, randlink, ccmapfile1, params) else: feed = findEMString(100+classavg, templatename, dwnimgname, ccmapfile1, params) sys.exit() execFindEM(feed) #Second round: template x template numstr = "%03d" % (200+classavg) ccmapfile2 = "cccmaxmap"+numstr+".mrc" apFile.removeFile(ccmapfile2) params["startang"+str(200+classavg)] = params["startang"+str(classavg)] params["endang"+str(200+classavg)] = params["endang"+str(classavg)] params["incrang"+str(200+classavg)] = params["incrang"+str(classavg)] feed = findEMString(200+classavg, templatename, templatename, ccmapfile2, params) execFindEM(feed) #Final round: (template x template) x (template x image) = spectral numstr = "%03d" % (300+classavg) ccmapfile3 = "cccmaxmap"+numstr+".mrc" apFile.removeFile(ccmapfile3) params["startang"+str(300+classavg)] = 0 params["endang"+str(300+classavg)] = 10 params["incrang"+str(300+classavg)] = 20 feed = findEMString(300+classavg, ccmapfile2, ccmapfile1, ccmapfile3, params) execFindEM(feed) #READ OUTPUT FILE if not os.path.isfile(ccmapfile3): apDisplay.printError("findem.exe did not run or crashed.\n"+ "Did you source useappion.sh?") else: ccmaxmap = apImage.mrcToArray(ccmapfile3) ccmaplist.append(ccmaxmap) return ccmaplist
def processImage(self, imgdata): self.ctfvalues = {} bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True) apix = apDatabase.getPixelSize(imgdata) if (not (self.params['onepass'] and self.params['zeropass'])): maskhighpass = False ace2inputpath = os.path.join(imgdata['session']['image path'], imgdata['filename'] + ".mrc") else: maskhighpass = True filterimg = apImage.maskHighPassFilter(imgdata['image'], apix, 1, self.params['zeropass'], self.params['onepass']) ace2inputpath = os.path.join(self.params['rundir'], imgdata['filename'] + ".mrc") mrc.write(filterimg, ace2inputpath) # make sure that the image is a square dimx = imgdata['camera']['dimension']['x'] dimy = imgdata['camera']['dimension']['y'] if dimx != dimy: dims = [dimx, dimy] dims.sort() apDisplay.printMsg("resizing image: %ix%i to %ix%i" % (dimx, dimy, dims[0], dims[0])) mrcarray = apImage.mrcToArray(ace2inputpath, msg=False) clippedmrc = apImage.frame_cut(mrcarray, [dims[0], dims[0]]) ace2inputpath = os.path.join(self.params['rundir'], imgdata['filename'] + ".mrc") apImage.arrayToMrc(clippedmrc, ace2inputpath, msg=False) ### pad out image to speed up FFT calculations for non-standard image sizes print "checking prime factor" if primefactor.isGoodStack(dimx) is False: goodsize = primefactor.getNextEvenPrime(dimx) factor = float(goodsize) / float(dimx) apDisplay.printMsg("padding image: %ix%i to %ix%i" % (dimx, dimy, dimx * factor, dimy * factor)) mrcarray = apImage.mrcToArray(ace2inputpath, msg=False) # paddedmrc = imagefun.pad(mrcarray, None, factor) paddedmrc = apImage.frame_constant(mrcarray, (dimx * factor, dimy * factor), cval=mrcarray.mean()) ace2inputpath = os.path.join(self.params['rundir'], imgdata['filename'] + ".mrc") apImage.arrayToMrc(paddedmrc, ace2inputpath, msg=False) inputparams = { 'input': ace2inputpath, 'cs': self.params['cs'], 'kv': imgdata['scope']['high tension'] / 1000.0, 'apix': apix, 'binby': self.params['bin'], } ### make standard input for ACE 2 apDisplay.printMsg("Ace2 executable: " + self.ace2exe) commandline = (self.ace2exe + " -i " + str(inputparams['input']) + " -b " + str(inputparams['binby']) + " -c " + str(inputparams['cs']) + " -k " + str(inputparams['kv']) + " -a " + str(inputparams['apix']) + " -e " + str(self.params['edge_b']) + "," + str(self.params['edge_t']) + " -r " + str(self.params['rotblur']) + "\n") ### run ace2 apDisplay.printMsg("running ace2 at " + time.asctime()) apDisplay.printColor(commandline, "purple") t0 = time.time() if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() ### check if ace2 worked basename = os.path.basename(ace2inputpath) imagelog = basename + ".ctf.txt" if not os.path.isfile(imagelog) and self.stats['count'] <= 1: ### ace2 always crashes on first image??? .fft_wisdom file?? time.sleep(1) if self.params['verbose'] is True: ace2proc = subprocess.Popen(commandline, shell=True) else: aceoutf = open("ace2.out", "a") aceerrf = open("ace2.err", "a") ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf) ace2proc.wait() if self.params['verbose'] is False: aceoutf.close() aceerrf.close() if not os.path.isfile(imagelog): lddcmd = "ldd " + self.ace2exe lddproc = subprocess.Popen(lddcmd, shell=True) lddproc.wait() apDisplay.printError("ace2 did not run") apDisplay.printMsg("ace2 completed in " + apDisplay.timeString(time.time() - t0)) ### parse log file self.ctfvalues = { 'cs': self.params['cs'], 'volts': imgdata['scope']['high tension'], } logf = open(imagelog, "r") apDisplay.printMsg("reading log file %s" % (imagelog)) for line in logf: sline = line.strip() if re.search("^Final Defocus: ", sline): ### old ACE2 apDisplay.printError( "This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now" ) #parts = sline.split() #self.ctfvalues['defocus1'] = float(parts[2]) #self.ctfvalues['defocus2'] = float(parts[3]) ### convert to degrees #self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4])) elif re.search("^Final Defocus \(m,m,deg\):", sline): ### new ACE2 apDisplay.printMsg("Reading new ACE2 defocus") parts = sline.split() #print parts self.ctfvalues['defocus1'] = float(parts[3]) self.ctfvalues['defocus2'] = float(parts[4]) # ace2 defines negative angle from +x toward +y self.ctfvalues['angle_astigmatism'] = -float(parts[5]) elif re.search("^Amplitude Contrast:", sline): parts = sline.split() self.ctfvalues['amplitude_contrast'] = float(parts[2]) elif re.search("^Confidence:", sline): parts = sline.split() self.ctfvalues['confidence'] = float(parts[1]) self.ctfvalues['confidence_d'] = float(parts[1]) logf.close() ### summary stats apDisplay.printMsg("============") avgdf = (self.ctfvalues['defocus1'] + self.ctfvalues['defocus2']) / 2.0 ampconst = 100.0 * self.ctfvalues['amplitude_contrast'] pererror = 100.0 * (self.ctfvalues['defocus1'] - self.ctfvalues['defocus2']) / avgdf apDisplay.printMsg( "Defocus: %.3f x %.3f um (%.2f percent astigmatism)" % (self.ctfvalues['defocus1'] * 1.0e6, self.ctfvalues['defocus2'] * 1.0e6, pererror)) apDisplay.printMsg("Angle astigmatism: %.2f degrees" % (self.ctfvalues['angle_astigmatism'])) apDisplay.printMsg("Amplitude contrast: %.2f percent" % (ampconst)) apDisplay.printColor( "Final confidence: %.3f" % (self.ctfvalues['confidence']), 'cyan') ### double check that the values are reasonable if avgdf > self.params['maxdefocus'] or avgdf < self.params[ 'mindefocus']: apDisplay.printWarning( "bad defocus estimate, not committing values to database") self.badprocess = True if ampconst < 0.0 or ampconst > 80.0: apDisplay.printWarning( "bad amplitude contrast, not committing values to database") self.badprocess = True if self.ctfvalues['confidence'] < 0.2: apDisplay.printWarning( "bad confidence value, not committing values to database") self.badprocess = True ## create power spectra jpeg mrcfile = imgdata['filename'] + ".mrc.edge.mrc" if os.path.isfile(mrcfile): jpegfile = os.path.join( self.powerspecdir, apDisplay.short(imgdata['filename']) + ".jpg") ps = apImage.mrcToArray(mrcfile, msg=False) c = numpy.array(ps.shape) / 2.0 ps[c[0] - 0, c[1] - 0] = ps.mean() ps[c[0] - 1, c[1] - 0] = ps.mean() ps[c[0] - 0, c[1] - 1] = ps.mean() ps[c[0] - 1, c[1] - 1] = ps.mean() #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) ps = numpy.log(ps + 1.0) ps = (ps - ps.mean()) / ps.std() cutoff = -2.0 * ps.min() ps = numpy.where(ps > cutoff, cutoff, ps) cutoff = ps.mean() ps = numpy.where(ps < cutoff, cutoff, ps) #print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max()) apImage.arrayToJpeg(ps, jpegfile, msg=False) apFile.removeFile(mrcfile) self.ctfvalues['graph3'] = jpegfile otherfiles = glob.glob(imgdata['filename'] + ".*.txt") ### remove extra debugging files for filename in otherfiles: if filename[-9:] == ".norm.txt": continue elif filename[-8:] == ".ctf.txt": continue else: apFile.removeFile(filename) if maskhighpass and os.path.isfile(ace2inputpath): apFile.removeFile(ace2inputpath) return
def processImage(self, imgdata): """ time ./ctffind3.exe << eof Input image file name [input.mrc] : 15aug13neil2_14jul14d_05sq_012hl_02ed-a.mrc Output diagnostic filename [diagnostic_output.mrc] : 15aug13neil2_14jul14d_05sq_012hl_02ed-a-pow.mrc Pixel size [1.0] : 2.7 Acceleration voltage [300.0] : 300 Spherical aberration [2.7] : 2.7 Amplitude contrast [0.07] : 0.07 Size of power spectrum to compute [512] : 512 Minimum resolution [30.0] : 20 Maximum resolution [5.0] : 5 Minimum defocus [5000.0] : Maximum defocus [50000.0] : Defocus search step [500.0] : Expected (tolerated) astigmatism [100.0] : Find additional phase shift? [no] : """ paramInputOrder = ['input', 'output', 'apix', 'kv', 'cs', 'ampcontrast', 'fieldsize', 'resmin', 'resmax', 'defmin', 'defmax', 'defstep', 'expect_astig', 'phase', 'newline',] #get Defocus in Angstroms self.ctfvalues = {} nominal = abs(imgdata['scope']['defocus']*-1.0e10) ctfvalue = ctfdb.getBestCtfByResolution(imgdata) if ctfvalue is not None: """ ## CTFFIND V3.5 (7-March-2012) prefers the smaller of the two values for astigmatic images I found that say you have an image with 1.1um and 1.5um defocus astigmatism. If you give CTFFIND the average value of 1.3um for the defocus and 0.4um astig (dast) then it will try to fit 1.3um and 1.8um, so you need to give it the minimum value (1.1um) for it to fit 1.1um and 1.5um. """ bestdef = min(ctfvalue['defocus1'],ctfvalue['defocus2'])*1.0e10 else: bestdef = nominal if ctfvalue is not None and self.params['bestdb'] is True: bestampcontrast = round(ctfvalue['amplitude_contrast'],3) beststigdiff = round(abs(ctfvalue['defocus1'] - ctfvalue['defocus2'])*1e10,1) else: bestampcontrast = self.params['ampcontrast'] beststigdiff = self.params['dast']*10000. if ctfvalue is not None and self.params['bestdb'] is True: ### set res max from resolution_80_percent gmean = (ctfvalue['resolution_80_percent']*ctfvalue['resolution_50_percent']*self.params['resmax'])**(1/3.) if gmean < self.params['resmin']*0.9: # replace only if valid Issue #3291 self.params['resmax'] = round(gmean,2) apDisplay.printColor("Setting resmax to the geometric mean of resolution values", "purple") # dstep is the physical detector pixel size apix = apDatabase.getPixelSize(imgdata) # inputparams defocii and astig are in Angstroms inputparams = { 'orig': os.path.join(imgdata['session']['image path'], imgdata['filename']+".mrc"), 'input': apDisplay.short(imgdata['filename'])+".mrc", 'output': apDisplay.short(imgdata['filename'])+"-pow.mrc", 'apix': apix, 'kv': imgdata['scope']['high tension']/1000.0, 'cs': self.params['cs'], 'ampcontrast': bestampcontrast, 'fieldsize': self.params['fieldsize'], 'resmin': self.params['resmin'], 'resmax': self.params['resmax'], 'defstep': self.params['defstep']*10000., #round(defocus/32.0, 1), 'expect_astig': beststigdiff, 'phase': 'no', # this is a secondary amp contrast term for phase plates 'newline': '\n', } defrange = self.params['defstep'] * self.params['numstep'] * 1e4 ## do 25 steps in either direction # in angstrum inputparams['defmin']= round(bestdef-defrange, 1) #in angstrom if inputparams['defmin'] < 0: apDisplay.printWarning("Defocus minimum is less than zero") inputparams['defmin'] = inputparams['defstep'] inputparams['defmax']= round(bestdef+defrange, 1) #in angstrom apDisplay.printColor("Defocus search range: %d A to %d A (%.2f to %.2f um)" %(inputparams['defmin'], inputparams['defmax'], inputparams['defmin']*1e-4, inputparams['defmax']*1e-4), "cyan") ### secondary lock check right before it starts on the real part if self.params['parallel'] and os.path.isfile(apDisplay.short(imgdata['filename'])+".mrc"): # This is a secondary image lock check, checking the first output of the process. # It alone is not good enough apDisplay.printWarning('Some other parallel process is working on the same image. Skipping') return ### create local link to image if not os.path.exists(inputparams['input']): os.symlink(inputparams['orig'], inputparams['input']) if os.path.isfile(inputparams['output']): # program crashes if this file exists apFile.removeFile(inputparams['output']) t0 = time.time() apDisplay.printMsg("running ctf estimation at "+time.asctime()) for paramName in paramInputOrder: apDisplay.printColor("%s = %s"%(paramName,inputparams[paramName]),"magenta") print "" ctfprogproc = subprocess.Popen(self.ctfprgmexe, shell=True, stdin=subprocess.PIPE,) apDisplay.printColor(self.ctfprgmexe, "magenta") for paramName in paramInputOrder: apDisplay.printColor(inputparams[paramName],"magenta") ctfprogproc.stdin.write(str(inputparams[paramName])+'\n') ctfprogproc.communicate() tdiff = time.time()-t0 apDisplay.printMsg("ctf estimation completed in "+apDisplay.timeString(tdiff)) if tdiff < 1.0: apDisplay.printError("Failed to run CTFFIND4 program...") ### cannot run ctffind_plot_results.sh on CentOS 6 # This script requires gnuplot version >= 4.6, but you have version 4.2 ### parse ctf estimation output self.ctfvalues = {} ctfproglog = apDisplay.short(imgdata['filename'])+"-pow.txt" apDisplay.printMsg("reading %s"%(ctfproglog)) logf = open(ctfproglog, "r") for line in logf: sline = line.strip() if sline.startswith('#'): continue bits = sline.split() self.ctfvalues = { 'imagenum': int(float(bits[0])), 'defocus2': float(bits[1])*1e-10, 'defocus1': float(bits[2])*1e-10, 'angle_astigmatism': float(bits[3]), 'extra_phase': float(bits[4]), 'amplitude_contrast': inputparams['ampcontrast'], 'cross_correlation': float(bits[5]), 'ctffind4_resolution': float(bits[6]), 'defocusinit': bestdef*1e-10, 'cs': self.params['cs'], 'volts': imgdata['scope']['high tension'], 'confidence': float(bits[5]), 'confidence_d': round(math.sqrt(abs(float(bits[5]))), 5) } print self.ctfvalues #convert powerspectra to JPEG outputjpgbase = apDisplay.short(imgdata['filename'])+"-pow.jpg" self.lastjpg = outputjpgbase outputjpg = os.path.join(self.powerspecdir, self.lastjpg) powspec = apImage.mrcToArray(inputparams['output']) apImage.arrayToJpeg(powspec, outputjpg) shutil.move(inputparams['output'], os.path.join(self.powerspecdir, inputparams['output'])) self.ctfvalues['graph1'] = outputjpg #apFile.removeFile(inputparams['input']) return
def tiltPhaseFlipParticles(self, imgdata, imgstackfile, partdatas): apDisplay.printMsg("Applying per-particle CTF") ctfvalue = ctfdb.getBestTiltCtfValueForImage(imgdata) if ctfvalue is None: apDisplay.printError("Failed to get ctf parameters") apix = apDatabase.getPixelSize(imgdata) ctfimgstackfile = os.path.join(self.params['rundir'], apDisplay.short(imgdata['filename'])+"-ctf.hed") ampconst = ctfvalue['amplitude_contrast'] ### calculate defocus at given position dimx = imgdata['camera']['dimension']['x'] dimy = imgdata['camera']['dimension']['y'] CX = dimx/2 CY = dimy/2 if ctfvalue['tilt_axis_angle'] is not None: N1 = -1.0 * math.sin( math.radians(ctfvalue['tilt_axis_angle']) ) N2 = math.cos( math.radians(ctfvalue['tilt_axis_angle']) ) else: N1 = 0.0 N2 = 1.0 PSIZE = apix ### High tension on CM is given in kv instead of v so do not divide by 1000 in that case if imgdata['scope']['tem']['name'] == "CM": voltage = imgdata['scope']['high tension'] else: voltage = (imgdata['scope']['high tension'])/1000 # find cs cs = self.getCS(ctfvalue) imagicdata = apImagicFile.readImagic(imgstackfile, msg=False) ctfpartstack = [] for i in range(len(partdatas)): partdata = partdatas[i] prepartarray = imagicdata['images'][i] prepartmrc = "rawpart.dwn.mrc" postpartmrc = "ctfpart.dwn.mrc" apImage.arrayToMrc(prepartarray, prepartmrc, msg = False) ### calculate ctf based on position NX = partdata['xcoord'] NY = dimy-partdata['ycoord'] # reverse due to boxer flip DX = CX - NX DY = CY - NY DF = (N1*DX + N2*DY) * PSIZE * math.tan( math.radians(ctfvalue['tilt_angle']) ) ### defocus is in Angstroms DFL1 = abs(ctfvalue['defocus1'])*1.0e10 + DF DFL2 = abs(ctfvalue['defocus2'])*1.0e10 + DF DF_final = (DFL1+DFL2)/2.0 ### convert defocus to microns defocus = DF_final*-1.0e-4 ### check to make sure defocus is a reasonable value for applyctf self.checkDefocus(defocus, apDisplay.short(imgdata['filename'])) parmstr = ("parm=%f,200,1,%.3f,0,17.4,9,1.53,%i,%.1f,%f" %(defocus, ampconst, voltage, cs, apix)) emancmd = ("applyctf %s %s %s setparm flipphase" % (prepartmrc, postpartmrc, parmstr)) apEMAN.executeEmanCmd(emancmd, showcmd = False) ctfpartarray = apImage.mrcToArray(postpartmrc, msg=False) ctfpartstack.append(ctfpartarray) apImagicFile.writeImagic(ctfpartstack, ctfimgstackfile) return ctfimgstackfile
def runFindEM(imgdict, params, thread=False): """ runs a separate thread of findem.exe for each template to get cross-correlation maps """ ### check image processAndSaveImage(imgdict, params) dwnimgname = imgdict['filename'] + ".dwn.mrc" if not os.path.isfile(dwnimgname): apDisplay.printError("cound not find image to process: " + dwnimgname) ### FindEM crashes when an input image is longer than 76 characters if len(dwnimgname) > 76: randlink = ''.join( random.choice(string.ascii_uppercase + string.digits) for x in range(10)) randlink += '.mrc' os.symlink(dwnimgname, randlink) ### check template if len(params['templatelist']) < 1: apDisplay.printError("templatelist == 0; there are no templates") joblist = [] ccmapfilelist = [] t0 = time.time() for i, templatename in enumerate(params['templatelist']): classavg = i + 1 #DETERMINE OUTPUT FILE NAME #CHANGE THIS TO BE 00%i in future numstr = "%03d" % classavg #numstr = str(classavg%10)+"00" ccmapfile = "cccmaxmap" + numstr + ".mrc" apFile.removeFile(ccmapfile) #GET FINDEM RUN COMMANDS if len(dwnimgname) > 76: feed = findEMString(classavg, templatename, randlink, ccmapfile, params) else: feed = findEMString(classavg, templatename, dwnimgname, ccmapfile, params) #RUN THE PROGRAM if thread is True: job = findemjob(feed) joblist.append(job) job.start() else: execFindEM(feed) #STORE OUTPUT FILE ccmapfilelist.append(ccmapfile) # get findemexe name findemexepath = getFindEMPath() findemexename = os.path.basename(findemexepath) ### WAIT FOR THREADS TO COMPLETE if thread is True: apDisplay.printMsg("waiting for " + str(len(joblist)) + " findem threads to complete") numtimes = 0 for i, job in enumerate(joblist): while job.isAlive(): sys.stderr.write(".") time.sleep(1.5) numtimes += 1 if numtimes == 40: pidof = "pidof %s" % findemexename pids = subprocess.Popen( pidof, shell=True, stdout=subprocess.PIPE).stdout.read() pids = pids.split() for pid in pids: os.kill(int(float(pid)), 9) apDisplay.printWarning( "\nFindEM likely stalled. Re-running the template correlator command\n" ) runFindEM(imgdict, params, thread) sys.stderr.write("\n") apDisplay.printMsg("FindEM finished in " + apDisplay.timeString(time.time() - t0)) ### READ OUTPUT FILES ccmaplist = [] for ccmapfile in ccmapfilelist: if not os.path.isfile(ccmapfile): apDisplay.printError("findem.exe did not run or crashed.\n") ccmaxmap = apImage.mrcToArray(ccmapfile) ccmaplist.append(ccmaxmap) return ccmaplist
def processImage(self, imgdata): """ time ./ctffind3.exe << eof micrograph.mrc montage.pow 2.0, 200.0, 0.07, 60000, 7.0, 2 #! CS[mm], HT[kV], AmpCnst, XMAG, DStep[um], PAve 128, 200.0, 8.0, 5000.0, 40000.0, 5000.0 #! Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep eof CARD 1: Input file name for image CARD 2: Output file name to check result CARD 3: CS[mm], HT[kV], AmpCnst, XMAG, DStep[um],PAve CARD 4: Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep, dAst[A] CTFTILT also asks for TiltA[deg], TiltR[deg] at CARD4 The output image file to check the result of the fitting shows the filtered average power spectrum of the input image in one half, and the fitted CTF (squared) in the other half. The two halves should agree very well for a successful fit. CS: Spherical aberration coefficient of the objective in mm HT: Electron beam voltage in kV AmpCnst: Amount of amplitude contrast (fraction). For ice images 0.07, for negative stain about 0.15. XMAG: Magnification of original image DStep: Pixel size on scanner in microns, or apix*mag/10000 PAve: Pixel averaging (PAve x PAve) for input image Box: Tile size. The program divides the image into square tiles and calculates the average power spectrum. Tiles with a significantly higher or lower variance are excluded; these are parts of the image which are unlikely to contain useful information (beam edge, film number etc). IMPORTANT: Box must have a even pixel dimensions. ResMin: Low resolution end of data to be fitted. ResMaX: High resolution end of data to be fitted. dFMin: Starting defocus value for grid search in Angstrom. Positive values represent an underfocus. The program performs a systematic grid search of defocus values and astigmatism before fitting a CTF to machine precision. dFMax: End defocus value for grid search in Angstrom. FStep: Step width for grid search in Angstrom. dAst: An additional parameter, dAst, was added to CARD 4 to restrain the amount of astigmatism in the CTF fit. This makes the fitting procedure more robust, especially in cases where the Thon rings are not easily visible. TiltA: guessed tilt angle TiltR: angular range for initial coarse search """ #get Defocus in Angstroms self.ctfvalues = {} nominal = abs(imgdata['scope']['defocus'] * -1.0e10) ctfvalue = ctfdb.getBestCtfByResolution(imgdata) if ctfvalue is not None: bestdef = abs(ctfvalue['defocus1'] + ctfvalue['defocus2']) / 2.0 * 1.0e10 else: bestdef = nominal if ctfvalue is not None and self.params['bestdb'] is True: bestampcontrast = ctfvalue['amplitude_contrast'] beststigdiff = abs(ctfvalue['defocus1'] - ctfvalue['defocus2']) * 1e10 else: bestampcontrast = self.params['amp' + self.params['medium']] beststigdiff = self.params['dast'] # dstep is the physical detector pixel size dstep = None if 'camera' in imgdata and imgdata['camera'] and imgdata['camera'][ 'pixel size']: dstep = imgdata['camera']['pixel size']['x'] if dstep is None: dstep = apDatabase.getPixelSize( imgdata) * imgdata['scope']['magnification'] / 10000.0 dstep /= 1e6 dstep = float(dstep) mpixelsize = apDatabase.getPixelSize(imgdata) * 1e-10 xmag = dstep / mpixelsize apDisplay.printMsg("Xmag=%d, dstep=%.2e, mpix=%.2e" % (xmag, dstep, mpixelsize)) inputparams = { 'orig': os.path.join(imgdata['session']['image path'], imgdata['filename'] + ".mrc"), 'input': apDisplay.short(imgdata['filename']) + ".mrc", 'output': apDisplay.short(imgdata['filename']) + "-pow.mrc", 'cs': self.params['cs'], 'kv': imgdata['scope']['high tension'] / 1000.0, 'ampcnst': bestampcontrast, 'xmag': xmag, 'dstep': dstep * 1e6, 'pixavg': self.params['bin'], 'box': self.params['fieldsize'], 'resmin': self.params['resmin'], 'resmax': self.params['resmax'], 'defstep': self.params['defstep'], #round(defocus/32.0, 1), 'dast': beststigdiff, } defrange = self.params['defstep'] * self.params[ 'numstep'] ## do 25 steps in either direction inputparams['defmin'] = round(bestdef - defrange, 1) #in meters if inputparams['defmin'] < 0: apDisplay.printWarning("Defocus minimum is less than zero") inputparams['defmin'] = inputparams['defstep'] inputparams['defmax'] = round(bestdef + defrange, 1) #in meters apDisplay.printColor( "Defocus search range: %d A to %d A (%.2f to %.2f um)" % (inputparams['defmin'], inputparams['defmax'], inputparams['defmin'] * 1e-4, inputparams['defmax'] * 1e-4), "cyan") ### create local link to image if not os.path.exists(inputparams['input']): cmd = "ln -s " + inputparams['orig'] + " " + inputparams[ 'input'] + "\n" proc = subprocess.Popen(cmd, shell=True) proc.wait() ### make standard input for ctf estimation line1cmd = inputparams['input'] + "\n" line2cmd = inputparams['output'] + "\n" line3cmd = (str(inputparams['cs']) + "," + str(inputparams['kv']) + "," + str(inputparams['ampcnst']) + "," + str(inputparams['xmag']) + "," + str(inputparams['dstep']) + "," + str(inputparams['pixavg']) + "\n") line4cmd = (str(inputparams['box']) + "," + str(inputparams['resmin']) + "," + str(inputparams['resmax']) + "," + str(inputparams['defmin']) + "," + str(inputparams['defmax']) + "," + str(inputparams['defstep']) + "," + str(inputparams['dast'])) ### additional ctftilt parameters if self.params['ctftilt'] is True: tiltang = apDatabase.getTiltAngleDeg(imgdata) line4cmd += ("," + str(tiltang) + ",10") line4cmd += "\n" if os.path.isfile(inputparams['output']): # program crashes if this file exists apFile.removeFile(inputparams['output']) t0 = time.time() apDisplay.printMsg("running ctf estimation at " + time.asctime()) ctfproglog = os.path.join( self.logdir, os.path.splitext(imgdata['filename'])[0] + "-ctfprog.log") logf = open(ctfproglog, "w") ctfprogproc = subprocess.Popen(self.ctfprgmexe, shell=True, stdin=subprocess.PIPE, stdout=logf) apDisplay.printColor(self.ctfprgmexe, "magenta") apDisplay.printColor(line1cmd.strip(), "magenta") apDisplay.printColor(line2cmd.strip(), "magenta") apDisplay.printColor(line3cmd.strip(), "magenta") apDisplay.printColor(line4cmd.strip(), "magenta") ctfprogproc.stdin.write(line1cmd) ctfprogproc.stdin.write(line2cmd) ctfprogproc.stdin.write(line3cmd) ctfprogproc.stdin.write(line4cmd) ctfprogproc.communicate() logf.close() apDisplay.printMsg("ctf estimation completed in " + apDisplay.timeString(time.time() - t0)) #apFile.removeFile(inputparams['input']) ### parse ctf estimation output self.ctfvalues = {} logf = open(ctfproglog, "r") ## ctffind & ctftilt have diff # values numvals = 6 if self.params['ctftilt'] is True: numvals = 8 for line in logf: sline = line.strip() if sline[-12:] == "Final Values": #print sline if '**********' in sline: sline = re.sub('**********', ' **********', sline) bits = sline.split() if len(bits) != numvals: apDisplay.printError("wrong number of values in " + str(bits)) for i, bit in enumerate(bits[0:(numvals - 2)]): bits[i] = float(bit) self.ctfvalues = { 'defocus1': float(bits[0]) * 1e-10, 'defocus2': float(bits[1]) * 1e-10, # WARNING: this is the negative of the direct result 'angle_astigmatism': float(bits[2]), 'amplitude_contrast': inputparams['ampcnst'], 'cross_correlation': float(bits[numvals - 3]), 'nominal': nominal * 1e-10, 'defocusinit': bestdef * 1e-10, 'cs': self.params['cs'], 'volts': imgdata['scope']['high tension'], 'confidence': float(bits[numvals - 3]), 'confidence_d': round(math.sqrt(abs(float(bits[numvals - 3]))), 5) } if self.params['ctftilt'] is True: self.ctfvalues['tilt_axis_angle'] = float(bits[3]) self.ctfvalues['tilt_angle'] = float(bits[4]) ### write to log file f = open("ctfvalues.log", "a") f.write("=== " + imgdata['filename'] + " ===\n") line1 = ("nominal=%.1e, bestdef=%.1e," % (self.ctfvalues['nominal'], self.ctfvalues['defocusinit'])) if self.params['ctftilt'] is True: self.ctfvalues['origtiltang'] = tiltang line1 += " tilt=%.1f," % tiltang apDisplay.printMsg(line1) f.write(line1) line2 = ( "def_1=%.1e, def_2=%.1e, astig_angle=%.1f, cross_corr=%.3f,\n" % (self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'], self.ctfvalues['cross_correlation'])) if self.params['ctftilt'] is True: line2 += ("tilt_angle=%.1f, tilt_axis_angle=%.1f,\n" % (self.ctfvalues['tilt_angle'], self.ctfvalues['tilt_axis_angle'])) apDisplay.printMsg(line2) f.write(line2) f.close() #convert powerspectra to JPEG outputjpgbase = os.path.basename( os.path.splitext(inputparams['output'])[0] + ".jpg") self.lastjpg = outputjpgbase outputjpg = os.path.join(self.powerspecdir, self.lastjpg) powspec = apImage.mrcToArray(inputparams['output']) apImage.arrayToJpeg(powspec, outputjpg) shutil.move(inputparams['output'], os.path.join(self.powerspecdir, inputparams['output'])) self.ctfvalues['graph1'] = outputjpg #apFile.removeFile(inputparams['input']) return
def runFindEM(imgdict, params, thread=False): """ runs a separate thread of findem.exe for each template to get cross-correlation maps """ ### check image processAndSaveImage(imgdict, params) dwnimgname = imgdict['filename']+".dwn.mrc" if not os.path.isfile(dwnimgname): apDisplay.printError("cound not find image to process: "+dwnimgname) ### FindEM crashes when an input image is longer than 76 characters if len(dwnimgname) > 76: randlink = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(10)) randlink+= '.mrc' os.symlink(dwnimgname, randlink) ### check template if len(params['templatelist']) < 1: apDisplay.printError("templatelist == 0; there are no templates") joblist = [] ccmapfilelist = [] t0 = time.time() for i,templatename in enumerate(params['templatelist']): classavg = i + 1 #DETERMINE OUTPUT FILE NAME #CHANGE THIS TO BE 00%i in future numstr = "%03d" % classavg #numstr = str(classavg%10)+"00" ccmapfile="cccmaxmap"+numstr+".mrc" apFile.removeFile(ccmapfile) #GET FINDEM RUN COMMANDS if len(dwnimgname) > 76: feed = findEMString(classavg, templatename, randlink, ccmapfile, params) else: feed = findEMString(classavg, templatename, dwnimgname, ccmapfile, params) #RUN THE PROGRAM if thread is True: job = findemjob(feed) joblist.append(job) job.start() else: execFindEM(feed) #STORE OUTPUT FILE ccmapfilelist.append(ccmapfile) # get findemexe name findemexepath = getFindEMPath() findemexename = os.path.basename(findemexepath) ### WAIT FOR THREADS TO COMPLETE if thread is True: apDisplay.printMsg("waiting for "+str(len(joblist))+" findem threads to complete") numtimes = 0 for i,job in enumerate(joblist): while job.isAlive(): sys.stderr.write(".") time.sleep(1.5) numtimes+=1 if numtimes == 40: pidof = "pidof %s" % findemexename pids=subprocess.Popen(pidof, shell=True,stdout=subprocess.PIPE).stdout.read() pids = pids.split() for pid in pids: os.kill(int(float(pid)),9) apDisplay.printWarning("\nFindEM likely stalled. Re-running the template correlator command\n") runFindEM(imgdict,params,thread) sys.stderr.write("\n") apDisplay.printMsg("FindEM finished in "+apDisplay.timeString(time.time()-t0)) ### READ OUTPUT FILES ccmaplist = [] for ccmapfile in ccmapfilelist: if not os.path.isfile(ccmapfile): apDisplay.printError("findem.exe did not run or crashed.\n") ccmaxmap = apImage.mrcToArray(ccmapfile) ccmaplist.append(ccmaxmap) return ccmaplist