def preLoopFunctions(self): self.powerspecdir = os.path.join(self.params['rundir'], "opimages") apParam.createDirectory(self.powerspecdir, warning=False) self.ctfrun = None self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) return
def checkConflicts(self): if self.params['resmin'] > 50.0: apDisplay.printError("Please choose a higher resolution for resmin must be btw 10 and 50") if self.params['resmin'] < 10.0: apDisplay.printError("Please choose a lower resolution for resmin") if self.params['resmax'] > 15.0 or self.params['resmax'] > self.params['resmin']: apDisplay.printError("Please choose a higher resolution for resmax") if self.params['defstep'] < 0.0001 or self.params['defstep'] > 2.0: apDisplay.printError("Please keep the defstep between 0.0001 & 2 microns") ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) return
def checkConflicts(self): if not (self.params['medium'] == 'carbon' or self.params['medium'] == 'ice'): apDisplay.printError("medium can only be 'carbon' or 'ice'") if self.params['resmin'] < 20.0: apDisplay.printError("Please choose a lower resolution for resmin") if self.params['resmax'] > 15.0 or self.params['resmax'] > self.params['resmin']: apDisplay.printError("Please choose a higher resolution for resmax") if self.params['defstep'] < 1.0 or self.params['defstep'] > 10000.0: apDisplay.printError("Please keep the defstep between 1 & 10000 Angstroms") ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) return
def checkConflicts(self): if self.params["stackid"] is None: apDisplay.printError("stack id was not defined") if self.params["modelid"] is None: apDisplay.printError("model id was not defined") if self.params["runname"] is None: apDisplay.printError("missing a reconstruction name") if self.params["last"] is None: self.params["last"] = apStack.getNumberStackParticlesFromId(self.params["stackid"]) self.boxsize = apStack.getStackBoxsize(self.params["stackid"], msg=False) self.apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) if self.params["reconstackid"] is not None: reconboxsize = apStack.getStackBoxsize(self.params["reconstackid"], msg=False) reconapix = apStack.getStackPixelSizeFromStackId(self.params["reconstackid"]) refinenumpart = apStack.getNumberStackParticlesFromId(self.params["stackid"]) reconnumpart = apStack.getNumberStackParticlesFromId(self.params["reconstackid"]) if reconboxsize != self.boxsize: apDisplay.printError("Boxsize do not match for stacks") if reconapix != self.apix: apDisplay.printError("Pixelsize do not match for stacks") if refinenumpart != reconnumpart: apDisplay.printError("Number of particles do not match for stacks") maxmask = math.floor(self.apix * (self.boxsize - 10) / 2.0) if self.params["mask"] is None: apDisplay.printWarning("mask was not defined, setting to boxsize: %d" % (maxmask)) self.params["mask"] = maxmask if self.params["mask"] > maxmask: apDisplay.printWarning("mask was too big, setting to boxsize: %d" % (maxmask)) self.params["mask"] = maxmask if self.params["noctf"] is True: apDisplay.printWarning("Using no CTF method") self.params["wgh"] = -1.0 if self.params["nodes"] > 1 and self.params["cluster"] is False: apDisplay.printError("cluster mode must be enabled to run on more than 1 node") if self.params["ppn"] is None: if self.params["cluster"] is True: apDisplay.printError("you must define ppn for cluster mode") self.params["ppn"] = apParam.getNumProcessors() apDisplay.printMsg("Setting number of processors to %d" % (self.params["ppn"])) if self.params["rpn"] is None: self.params["rpn"] = self.params["ppn"] self.params["nproc"] = self.params["nodes"] * self.params["rpn"] ### get the symmetry data if self.params["sym"] is None: apDisplay.printError("Symmetry was not defined") else: self.symmdata = apSymmetry.findSymmetry(self.params["sym"]) self.params["symm_id"] = self.symmdata.dbid self.params["symm_name"] = self.symmdata["eman_name"] apDisplay.printMsg("Selected symmetry %s with id %s" % (self.symmdata["eman_name"], self.symmdata.dbid)) ### set cs value self.params["cs"] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['modelid'] is None: apDisplay.printError("model id was not defined") if self.params['runname'] is None: apDisplay.printError("missing a reconstruction name") if self.params['last'] is None: self.params['last'] = apStack.getNumberStackParticlesFromId(self.params['stackid']) self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False) self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) if self.params['reconstackid'] is not None: reconboxsize = apStack.getStackBoxsize(self.params['reconstackid'], msg=False) reconapix = apStack.getStackPixelSizeFromStackId(self.params['reconstackid']) refinenumpart = apStack.getNumberStackParticlesFromId(self.params['stackid']) reconnumpart = apStack.getNumberStackParticlesFromId(self.params['reconstackid']) if reconboxsize != self.boxsize: apDisplay.printError("Boxsize do not match for stacks") if reconapix != self.apix: apDisplay.printError("Pixelsize do not match for stacks") if refinenumpart != reconnumpart: apDisplay.printError("Number of particles do not match for stacks") maxmask = math.floor(self.apix*(self.boxsize-10)/2.0) if self.params['mask'] is None: apDisplay.printWarning("mask was not defined, setting to boxsize: %d"%(maxmask)) self.params['mask'] = maxmask if self.params['mask'] > maxmask: apDisplay.printWarning("mask was too big, setting to boxsize: %d"%(maxmask)) self.params['mask'] = maxmask if self.params['noctf'] is True: apDisplay.printWarning("Using no CTF method") self.params['wgh'] = -1.0 if self.params['nodes'] > 1 and self.params['cluster'] is False: apDisplay.printError("cluster mode must be enabled to run on more than 1 node") if self.params['ppn'] is None: if self.params['cluster'] is True: apDisplay.printError("you must define ppn for cluster mode") self.params['ppn'] = apParam.getNumProcessors() apDisplay.printMsg("Setting number of processors to %d"%(self.params['ppn'])) if self.params['rpn'] is None: self.params['rpn'] = self.params['ppn'] self.params['nproc'] = self.params['nodes']*self.params['rpn'] ### get the symmetry data if self.params['sym'] is None: apDisplay.printError("Symmetry was not defined") else: self.symmdata = apSymmetry.findSymmetry(self.params['sym']) self.params['symm_id'] = self.symmdata.dbid self.params['symm_name'] = self.symmdata['eman_name'] apDisplay.printMsg("Selected symmetry %s with id %s"%(self.symmdata['eman_name'], self.symmdata.dbid)) ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self): if self.params['bin'] < 1: apDisplay.printError("bin must be positive") if (self.params['mindefocus'] is not None and (self.params['mindefocus'] > 1e-3 or self.params['mindefocus'] < 1e-9)): apDisplay.printError("min defocus is not in an acceptable range, e.g. mindefocus=1.5e-6") if (self.params['maxdefocus'] is not None and (self.params['maxdefocus'] > 1e-3 or self.params['maxdefocus'] < 1e-9)): apDisplay.printError("max defocus is not in an acceptable range, e.g. maxdefocus=1.5e-6") ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) return
def checkConflicts(self): if self.params['nominal'] is not None and (self.params['nominal'] > 0 or self.params['nominal'] < -15e-6): apDisplay.printError("Nominal should be of the form nominal=-1.2e-6 for -1.2 microns") if not (self.params['drange'] == 1 or self.params['drange']== 0): apDisplay.printError("drange should only be 0 or 1") if not (self.params['medium'] == 'carbon' or self.params['medium'] == 'ice'): apDisplay.printError("medium can only be 'carbon' or 'ice'") if not (self.params['display'] == 0 or self.params['display'] == 1): apDisplay.printError("display must be 0 or 1") if not (self.params['stig'] == 0 or self.params['stig'] == 1): apDisplay.printError("stig must be 0 or 1") ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) return
def checkConflicts(self): if self.params["nominal"] is not None and (self.params["nominal"] > 0 or self.params["nominal"] < -15e-6): apDisplay.printError("Nominal should be of the form nominal=-1.2e-6 for -1.2 microns") if not (self.params["drange"] == 1 or self.params["drange"] == 0): apDisplay.printError("drange should only be 0 or 1") if not (self.params["medium"] == "carbon" or self.params["medium"] == "ice"): apDisplay.printError("medium can only be 'carbon' or 'ice'") if not (self.params["display"] == 0 or self.params["display"] == 1): apDisplay.printError("display must be 0 or 1") if not (self.params["stig"] == 0 or self.params["stig"] == 1): apDisplay.printError("stig must be 0 or 1") ### set cs value self.params["cs"] = apInstrument.getCsValueFromSession(self.getSessionData()) return
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stackid was not defined") if self.params['modelid'] is None: apDisplay.printError("model id was not defined") if self.params['runname'] is None: apDisplay.printError("new runname was not defined") self.params['totalpart'] = apStack.getNumberStackParticlesFromId(self.params['stackid']) if 'last' not in self.params.keys() or self.params['last'] is None: self.params['last'] = self.params['totalpart'] self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False) self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.refineboxsize = self.boxsize * self.params['bin'] ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) self.modelids = map((lambda x: int(x)),self.params['modelid'].split(',')) self.checkPackageConflicts()
def checkConflicts(self): if self.params['bin'] < 1: apDisplay.printError("bin must be positive") if (self.params['mindefocus'] is not None and (self.params['mindefocus'] > 1e-3 or self.params['mindefocus'] < 1e-9)): apDisplay.printError( "min defocus is not in an acceptable range, e.g. mindefocus=1.5e-6" ) if (self.params['maxdefocus'] is not None and (self.params['maxdefocus'] > 1e-3 or self.params['maxdefocus'] < 1e-9)): apDisplay.printError( "max defocus is not in an acceptable range, e.g. maxdefocus=1.5e-6" ) ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession( self.getSessionData()) return
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stackid was not defined") if self.params['modelid'] is None: apDisplay.printError("model id was not defined") if self.params['runname'] is None: apDisplay.printError("new runname was not defined") self.params['totalpart'] = apStack.getNumberStackParticlesFromId( self.params['stackid']) if 'last' not in self.params.keys() or self.params['last'] is None: self.params['last'] = self.params['totalpart'] self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False) self.apix = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.refineboxsize = self.boxsize * self.params['bin'] ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession( self.getSessionData()) self.modelids = map((lambda x: int(x)), self.params['modelid'].split(',')) self.checkPackageConflicts()
def processImage(self, imgdata): self.ctfvalues = {} ### convert image to spider spiderimage = apDisplay.short(imgdata['filename']) + ".spi" spider.write(imgdata['image'], spiderimage) ### high tension in kiloVolts kvolts = imgdata['scope']['high tension'] / 1000.0 ### spherical aberration in millimeters cs = apInstrument.getCsValueFromSession(self.getSessionData()) ### pixel size in Angstroms apix = apDatabase.getPixelSize(imgdata) ### write image name inputstr = "" inputstr += ("\n") inputstr += ("# image filename in spider format\n") inputstr += ("image=%s\n" % (spiderimage)) ### common parameters that never change inputstr += ("\n") inputstr += ("# common parameters that never change\n") inputstr += ("N_horizontal=%d\n" % (self.params['fieldsize'])) #inputstr += ("particle_horizontal=%d\n"%(128)) inputstr += ("show_optimization=yes\n") inputstr += ("micrograph_averaging=yes\n") """ Give a value in digital frequency (i.e. between 0.0 and 0.5) This cut-off prevents the typically peak at the center of the PSD to interfere with CTF estimation. The default value is 0.05, but for micrographs with a very fine sampling this may be lowered towards 0.0 """ #minres = apix/minfreq => minfreq = apix/minres minfreq = apix / self.params['resmin'] inputstr += ("min_freq=%.3f\n" % (minfreq)) """ Give a value in digital frequency (i.e. between 0.0 and 0.5) This cut-off prevents high-resolution terms where only noise exists to interfere with CTF estimation. The default value is 0.35, but it should be increased for micrographs with signals extending beyond this value """ #maxres = apix/maxfreq => maxfreq = apix/maxres maxfreq = apix / self.params['resmax'] inputstr += ("max_freq=%.3f\n" % (maxfreq)) ### CTF input parameters from database inputstr += ("\n") inputstr += ("# CTF input parameters from database\n") inputstr += ("voltage=%d\n" % (kvolts)) inputstr += ("spherical_aberration=%.1f\n" % (cs)) inputstr += ("sampling_rate=%.4f\n" % (apix)) ### best defocus in negative Angstroms ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata, msg=True) nominal = (ctfvalue['defocus1'] + ctfvalue['defocus2']) / 2.0 inputstr += ("defocusU=%d\n" % (-abs(ctfvalue['defocus1']) * 1.0e10)) inputstr += ("defocusV=%d\n" % (-abs(ctfvalue['defocus2']) * 1.0e10)) inputstr += ("Q0=%d\n" % (-ctfvalue['amplitude_contrast'])) inputstr += ("\n") ### open and write to input parameter file paramfile = apDisplay.short(imgdata['filename']) + "-CTF.prm" f = open(paramfile, "w") print inputstr f.write(inputstr) f.close() #[min_freq=<f=0.05>] #[max_freq=<f=0.35>] xmippcmd = "xmipp_ctf_estimate_from_micrograph -i %s" % (paramfile) #xmippcmd = "echo %s"%(paramfile) t0 = time.time() apDisplay.printMsg("running ctf estimation at " + time.asctime()) proc = subprocess.Popen(xmippcmd, shell=True, stdout=subprocess.PIPE) #waittime = 2 #while proc.poll() is None: # sys.stderr.write(".") # time.sleep(waittime) (stdout, stderr) = proc.communicate() #print (stdout, stderr) apDisplay.printMsg("ctf estimation completed in " + apDisplay.timeString(time.time() - t0)) ### read commandline output to get fit score lines = stdout.split('\n') lastline = "" for line in lines[-50:]: if "--->" in line: lastline = line if not "--->" in lastline: apDisplay.printWarning("Xmipp CTF failed") self.badprocess = True return bits = lastline.split('--->') confidence = float(bits[1]) score = round(math.sqrt(1 - confidence), 5) apDisplay.printColor( "Confidence value is %.5f (score %.5f)" % (confidence, score), "cyan") f = open("confidence.log", "a") f.write("Confidence value is %.5f for image %s (score %.3f)\n" % (confidence, apDisplay.short(imgdata['filename']), score)) f.close() ### delete image in spider format no longer needed apFile.removeFile(spiderimage) ### read output parameter file outparamfile = apDisplay.short( imgdata['filename']) + "_Periodogramavg.ctfparam" f = open(outparamfile, "r") for line in f: sline = line.strip() bits = sline.split('=') if sline.startswith("defocusU"): defocus1 = float(bits[1].strip()) elif sline.startswith("defocusV"): defocus2 = float(bits[1].strip()) elif sline.startswith("azimuthal_angle"): angle_astigmatism = float(bits[1].strip()) elif sline.startswith("Q0"): amplitude_contrast = abs(float(bits[1].strip())) print defocus1, defocus2, angle_astigmatism, amplitude_contrast #defocusU= -18418.6 #defocusV= -24272.1 #azimuthal_angle= 79.7936 #Q0= -0.346951 #negative of ACE amplitude_contrast print "AMP CONTRAST: %.4f -- %.4f" % (amplitude_contrast, ctfvalue['amplitude_contrast']) self.ctfvalues = { 'defocus1': defocus1 * 1e-10, 'defocus2': defocus2 * 1e-10, 'angle_astigmatism': angle_astigmatism, 'amplitude_contrast': amplitude_contrast, 'nominal': nominal, 'defocusinit': nominal, 'confidence_d': score, 'cs': self.params['cs'], 'volts': kvolts * 1000.0, } return
def runRefineCTF(self, imgdata, fftpath): ### reset important values self.bestres = 1e10 self.bestellipse = None self.bestvalues = {} self.volts = imgdata['scope']['high tension'] self.wavelength = ctftools.getTEMLambda(self.volts) ## get value in meters self.cs = apInstrument.getCsValueFromSession( self.getSessionData()) * 1e-3 self.ctfvalues = { 'volts': self.volts, 'wavelength': self.wavelength, 'cs': self.cs, } ### need to get FFT file open and freq of said file fftarray = mrc.read(fftpath).astype(numpy.float64) self.freq = self.freqdict[fftpath] ### convert resolution limit into pixel distance fftwidth = fftarray.shape[0] maxres = 2.0 / (self.freq * fftwidth) if maxres > self.params['reslimit']: apDisplay.printWarning( "Cannot get requested res %.1fA higher than max Nyquist resolution %.1fA" % (self.params['reslimit'], maxres)) self.params['reslimit'] = math.ceil(maxres) limitwidth = int(math.ceil(2.0 / (self.params['reslimit'] * self.freq))) limitwidth = primefactor.getNextEvenPrime(limitwidth) requestres = 2.0 / (self.freq * limitwidth) if limitwidth > fftwidth: apDisplay.printError("Cannot get requested resolution" + ( " request res %.1fA higher than max res %.1fA for new widths %d > %d" % (requestres, maxres, limitwidth, fftwidth))) apDisplay.printColor( "Requested resolution OK: " + (" request res %.1fA less than max res %.1fA with fft widths %d < %d" % (requestres, maxres, limitwidth, fftwidth)), "green") newshape = (limitwidth, limitwidth) fftarray = imagefilter.frame_cut(fftarray, newshape) ### spacing parameters self.mfreq = self.freq * 1e10 fftwidth = min(fftarray.shape) self.apix = 1.0 / (fftwidth * self.freq) self.ctfvalues['apix'] = self.apix ### print message bestDbValues = ctfdb.getBestCtfByResolution(imgdata) if bestDbValues is None: apDisplay.printColor( "SKIPPING: No CTF values for image %s" % (apDisplay.short(imgdata['filename'])), "red") self.badprocess = True return ### skip if resolution > 90. if bestDbValues['resolution_50_percent'] > 90.: apDisplay.printColor( "SKIPPING: No decent CTF values for image %s" % (apDisplay.short(imgdata['filename'])), "yellow") self.badprocess = True return self.ctfvalues['amplitude_contrast'] = bestDbValues[ 'amplitude_contrast'] lowerrad1 = ctftools.getCtfExtrema( bestDbValues['defocus1'], self.mfreq, self.cs, self.volts, self.ctfvalues['amplitude_contrast'], 1, "valley") lowerrad2 = ctftools.getCtfExtrema( bestDbValues['defocus2'], self.mfreq, self.cs, self.volts, self.ctfvalues['amplitude_contrast'], 1, "valley") meanRad = (lowerrad1[0] + lowerrad2[0]) / 2.0 self.ellipseParams = { 'a': lowerrad1[0], 'b': lowerrad2[0], 'alpha': math.radians(bestDbValues['angle_astigmatism']), } ellipratio = self.ellipseParams['a'] / self.ellipseParams['b'] defratio = bestDbValues['defocus2'] / bestDbValues['defocus1'] print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f" % (ellipratio, defratio, math.sqrt(defratio)) self.bestvalues['defocus'] = (bestDbValues['defocus1'] + bestDbValues['defocus2']) / 2.0 raddata, PSDarray = self.from2Dinto1D(fftarray) lowerbound = numpy.searchsorted(raddata, meanRad * self.freq) upperbound = numpy.searchsorted(raddata, 1 / 10.) ### # This is the start of the actual program ### self.refineEllipseLoop(fftarray, lowerbound, upperbound) ##================================== ## FINISH UP ##================================== apDisplay.printColor("Finishing up using best found CTF values", "blue") self.printBestValues() ### take best values and use them self.ctfvalues = self.bestvalues self.ellipseParams = self.bestellipse ### stupid fix, get value in millimeters self.ctfvalues['cs'] = apInstrument.getCsValueFromSession( self.getSessionData()) ### translate ellipse into ctf values if self.ellipseParams is not None: self.ctfvalues['angle_astigmatism'] = math.degrees( self.ellipseParams['alpha']) ellipratio = self.ellipseParams['a'] / self.ellipseParams['b'] phi = math.asin(self.ctfvalues['amplitude_contrast']) #note: a > b then def1 < def2 #major axis self.ctfvalues['defocus1'] = self.ctfvalues['defocus'] / ellipratio #minor axis self.ctfvalues['defocus2'] = self.ctfvalues['defocus'] * ellipratio defdiff = 1.0 - 2 * self.ctfvalues['defocus'] / ( self.ctfvalues['defocus1'] + self.ctfvalues['defocus2']) print "%.3e --> %.3e,%.3e" % (self.ctfvalues['defocus'], self.ctfvalues['defocus2'], self.ctfvalues['defocus1']) print defdiff * 100 if defdiff * 100 > 1: apDisplay.printWarning("Large astigmatism") #sys.exit(1) else: ellipratio = 1.0 self.ctfvalues['angle_astigmatism'] = 0.0 self.ctfvalues['defocus1'] = self.ctfvalues['defocus'] self.ctfvalues['defocus2'] = self.ctfvalues['defocus'] self.badprocess = True return try: if self.ctfvalues['amplitude_contrast'] < self.minAmpCon: self.ctfvalues['amplitude_contrast'] = self.minAmpCon if self.ctfvalues['amplitude_contrast'] > self.maxAmpCon: self.ctfvalues['amplitude_contrast'] = self.maxAmpCon except KeyError: pass if abs(self.ctfvalues['defocus1']) > abs(self.ctfvalues['defocus2']): # incorrect, need to shift angle by 90 degrees apDisplay.printWarning("|def1| > |def2|, flipping defocus axes") tempdef = self.ctfvalues['defocus1'] self.ctfvalues['defocus1'] = self.ctfvalues['defocus2'] self.ctfvalues['defocus2'] = tempdef self.ctfvalues['angle_astigmatism'] += 90 # get astig_angle within range -90 < angle <= 90 while self.ctfvalues['angle_astigmatism'] > 90: self.ctfvalues['angle_astigmatism'] -= 180 while self.ctfvalues['angle_astigmatism'] < -90: self.ctfvalues['angle_astigmatism'] += 180 avgres = self.getResolution(self.ctfvalues['defocus'], raddata, PSDarray, lowerbound) apDisplay.printColor( "Final defocus values %.3e -> %.3e, %.3e; ac=%.2f, res=%.1f" % (self.ctfvalues['defocus'], self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['amplitude_contrast'], avgres / 2.0), "green") for i in range(10): print "====================================" print "PREVIOUS VALUES" ctfdb.getBestCtfByResolution(imgdata) print "CURRENT VALUES" defocusratio = self.ctfvalues['defocus2'] / self.ctfvalues['defocus1'] apDisplay.printColor( "def1: %.2e | def2: %.2e | angle: %.1f | ampcontr %.2f | defratio %.3f" % (self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'], self.ctfvalues['amplitude_contrast'], defocusratio), "blue") self.printBestValues() #ellipratio = self.ellipseParams['a']/self.ellipseParams['b'] print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f" % ( ellipratio, defocusratio, math.sqrt(defocusratio)) print "====================================" return
def checkConflicts(self): self.appiondir = apParam.getAppionDirectory() ### necessary input values if self.params['threedfile'] is None and self.params['modelid'] is None: apDisplay.printError('either threed .mrc file or modelid was not defined') if self.params['threedfile'] is not None and self.params['modelid'] is not None: apDisplay.printError('please specify a single .mrc file (i.e. threedfile or modelid)') if self.params['box'] is None and self.params['modelid'] is None: apDisplay.printError('boxsize of the output stack not specified') if self.params['apix'] is None and self.params['modelid'] is None: apDisplay.printError('angstroms per pixel of the input model not specified') ### radius defaulted to 0.5 if self.params['radius'] is None: self.params['radius'] = self.params['box'] / 2 if self.params['radius'] > self.params['box']/2: apDisplay.printError('radius of particle (in pixels) must be smaller than 1/2*boxsize of model') if self.params['radius'] > 0 and self.params['radius'] < 1: apDisplay.printError('radius must be specified in pixels') ### get session info if self.params['sessionname'] is None: split = self.params['rundir'].split("/") self.params['sessionname'] = split[4] ### make sure that the defoci are positive (underfocus) and in microns self.params['df1'] *= 10**-6 self.params['df2'] *= 10**-6 if self.params['df1'] < 0: apDisplay.printError('defocus value is negative! specify positive (underfocus) values') if self.params['df2'] < 0: apDisplay.printError('defocus value is negative! specify positive (underfocus) values') ### check defocus group values if self.params['defocus_groups'] is not None: self.defmin = float(self.params['defocus_groups'].split(",")[0]) self.defmax = float(self.params['defocus_groups'].split(",")[1]) self.defint = float(self.params['defocus_groups'].split(",")[2]) if self.defmin is None or self.defmax is None or self.defint is None: apDisplay.printError("all defocus group values must be specified, e.g. --defocus_groups=1,3,0.1") ### make sure that only one type of ace2correction is specified if self.params['ace2correct'] is True and self.params['ace2correct_rand'] is True: apDisplay.printError('Please specify only 1 type of ace2 correction') if self.params['ace2correct_std'] >= 0.5 or self.params['ace2correct_std'] <= 0: apDisplay.printError("Ace2correct standard deviation specified too high, please use value between 0 < std < 0.5") if self.params['ace2estimate'] is True and self.params['ace2correct'] is False: apDisplay.printError("ACE2 estimation should only be used if you're doing correction as well, please use both ace2correct and ace2estimate") ### make sure amplitude correction file exists if self.params['envelopefile'] is None: self.params['envelopefile'] = os.path.join(apParam.getAppionDirectory(), "appionlib/data/radial-envelope.spi") ### set cs value & make sure that it's in millimeters if self.params['cs'] is None: self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) self.params['cs'] = self.params['cs'] if self.params['cs'] > 0: apDisplay.printWarning("non-zero cs value is untested and may not be properly applied. Consider setting to 0") apDisplay.printColor("cs value: %.3f" % self.params['cs'], "cyan") return
def runRefineCTF(self, imgdata, fftpath): ### reset important values self.bestres = 1e10 self.bestellipse = None self.bestvalues = {} self.volts = imgdata['scope']['high tension'] self.wavelength = ctftools.getTEMLambda(self.volts) ## get value in meters self.cs = apInstrument.getCsValueFromSession(self.getSessionData())*1e-3 self.ctfvalues = { 'volts': self.volts, 'wavelength': self.wavelength, 'cs': self.cs, } ### need to get FFT file open and freq of said file fftarray = mrc.read(fftpath).astype(numpy.float64) self.freq = self.freqdict[fftpath] ### convert resolution limit into pixel distance fftwidth = fftarray.shape[0] maxres = 2.0/(self.freq*fftwidth) if maxres > self.params['reslimit']: apDisplay.printError("Cannot get requested res %.1fA higher than max res %.1fA" %(maxres, self.params['reslimit'])) limitwidth = int(math.ceil(2.0/(self.params['reslimit']*self.freq))) limitwidth = primefactor.getNextEvenPrime(limitwidth) requestres = 2.0/(self.freq*limitwidth) if limitwidth > fftwidth: apDisplay.printError("Cannot get requested resolution" +(" request res %.1fA higher than max res %.1fA for new widths %d > %d" %(requestres, maxres, limitwidth, fftwidth))) apDisplay.printColor("Requested resolution OK: " +(" request res %.1fA less than max res %.1fA with fft widths %d < %d" %(requestres, maxres, limitwidth, fftwidth)), "green") newshape = (limitwidth, limitwidth) fftarray = imagefilter.frame_cut(fftarray, newshape) ### spacing parameters self.mfreq = self.freq*1e10 fftwidth = min(fftarray.shape) self.apix = 1.0/(fftwidth*self.freq) self.ctfvalues['apix'] = self.apix ### print message bestDbValues = ctfdb.getBestCtfByResolution(imgdata) if bestDbValues is None: apDisplay.printColor("SKIPPING: No CTF values for image %s" %(apDisplay.short(imgdata['filename'])), "red") self.badprocess = True return ### skip if resolution > 90. if bestDbValues['resolution_50_percent'] > 90.: apDisplay.printColor("SKIPPING: No decent CTF values for image %s" %(apDisplay.short(imgdata['filename'])), "yellow") self.badprocess = True return self.ctfvalues['amplitude_contrast'] = bestDbValues['amplitude_contrast'] lowerrad1 = ctftools.getCtfExtrema(bestDbValues['defocus1'], self.mfreq, self.cs, self.volts, self.ctfvalues['amplitude_contrast'], 1, "valley") lowerrad2 = ctftools.getCtfExtrema(bestDbValues['defocus2'], self.mfreq, self.cs, self.volts, self.ctfvalues['amplitude_contrast'], 1, "valley") meanRad = (lowerrad1[0] + lowerrad2[0])/2.0 self.ellipseParams = { 'a': lowerrad1[0], 'b': lowerrad2[0], 'alpha': math.radians(bestDbValues['angle_astigmatism']), } ellipratio = self.ellipseParams['a']/self.ellipseParams['b'] defratio = bestDbValues['defocus2']/bestDbValues['defocus1'] print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f"%(ellipratio, defratio, math.sqrt(defratio)) self.bestvalues['defocus'] = (bestDbValues['defocus1']+bestDbValues['defocus2'])/2.0 raddata, PSDarray = self.from2Dinto1D(fftarray) lowerbound = numpy.searchsorted(raddata, meanRad*self.freq) upperbound = numpy.searchsorted(raddata, 1/10.) ### # This is the start of the actual program ### self.refineEllipseLoop(fftarray, lowerbound, upperbound) ##================================== ## FINISH UP ##================================== apDisplay.printColor("Finishing up using best found CTF values", "blue") self.printBestValues() ### take best values and use them self.ctfvalues = self.bestvalues self.ellipseParams = self.bestellipse ### stupid fix, get value in millimeters self.ctfvalues['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) ### translate ellipse into ctf values if self.ellipseParams is not None: self.ctfvalues['angle_astigmatism'] = math.degrees(self.ellipseParams['alpha']) ellipratio = self.ellipseParams['a']/self.ellipseParams['b'] phi = math.asin(self.ctfvalues['amplitude_contrast']) #note: a > b then def1 < def2 #major axis self.ctfvalues['defocus1'] = self.ctfvalues['defocus']/ellipratio #minor axis self.ctfvalues['defocus2'] = self.ctfvalues['defocus']*ellipratio defdiff = 1.0 - 2*self.ctfvalues['defocus']/(self.ctfvalues['defocus1']+self.ctfvalues['defocus2']) print "%.3e --> %.3e,%.3e"%(self.ctfvalues['defocus'], self.ctfvalues['defocus2'], self.ctfvalues['defocus1']) print defdiff*100 if defdiff*100 > 1: apDisplay.printWarning("Large astigmatism") #sys.exit(1) else: ellipratio = 1.0 self.ctfvalues['angle_astigmatism'] = 0.0 self.ctfvalues['defocus1'] = self.ctfvalues['defocus'] self.ctfvalues['defocus2'] = self.ctfvalues['defocus'] if self.ctfvalues['amplitude_contrast'] < 0.0: self.ctfvalues['amplitude_contrast'] = 0.0 if self.ctfvalues['amplitude_contrast'] > self.maxAmpCon: self.ctfvalues['amplitude_contrast'] = self.maxAmpCon if abs(self.ctfvalues['defocus1']) > abs(self.ctfvalues['defocus2']): # incorrect, need to shift angle by 90 degrees apDisplay.printWarning("|def1| > |def2|, flipping defocus axes") tempdef = self.ctfvalues['defocus1'] self.ctfvalues['defocus1'] = self.ctfvalues['defocus2'] self.ctfvalues['defocus2'] = tempdef self.ctfvalues['angle_astigmatism'] += 90 # get astig_angle within range -90 < angle <= 90 while self.ctfvalues['angle_astigmatism'] > 90: self.ctfvalues['angle_astigmatism'] -= 180 while self.ctfvalues['angle_astigmatism'] < -90: self.ctfvalues['angle_astigmatism'] += 180 avgres = self.getResolution(self.ctfvalues['defocus'], raddata, PSDarray, lowerbound) apDisplay.printColor("Final defocus values %.3e -> %.3e, %.3e; ac=%.2f, res=%.1f" %(self.ctfvalues['defocus'], self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['amplitude_contrast'], avgres/2.0), "green") for i in range(10): print "====================================" print "PREVIOUS VALUES" ctfdb.getBestCtfByResolution(imgdata) print "CURRENT VALUES" defocusratio = self.ctfvalues['defocus2']/self.ctfvalues['defocus1'] apDisplay.printColor("def1: %.2e | def2: %.2e | angle: %.1f | ampcontr %.2f | defratio %.3f" %(self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'], self.ctfvalues['amplitude_contrast'], defocusratio), "blue") self.printBestValues() #ellipratio = self.ellipseParams['a']/self.ellipseParams['b'] print "ellr=%.3f, defr=%.3f, sqrt(defr)=%.3f"%(ellipratio, defocusratio, math.sqrt(defocusratio)) print "====================================" return
def runPhasorCTF(self, imgdata, fftpath): ### reset important values self.bestres = 1e10 self.bestellipse = None self.bestvalues = None self.ellipseParams = None self.volts = imgdata['scope']['high tension'] self.wavelength = ctftools.getTEMLambda(self.volts) ## get value in meters self.cs = apInstrument.getCsValueFromSession(self.getSessionData())*1e-3 self.ctfvalues = { 'volts': self.volts, 'wavelength': self.wavelength, 'cs': self.cs, } if self.params['astig'] is False: self.maxRatio=1.3 else: self.maxRatio=2.0 ### need to get FFT file open and freq of said file fftarray = mrc.read(fftpath).astype(numpy.float64) self.freq = self.freqdict[fftpath] ### print message ctfdb.getBestCtfByResolution(imgdata) ### convert resolution limit into pixel distance fftwidth = fftarray.shape[0] maxres = 2.0/(self.freq*fftwidth) if maxres > self.params['reslimit']: apDisplay.printError("Cannot get requested res %.1fA higher than max res %.1fA" %(maxres, self.params['reslimit'])) limitwidth = int(math.ceil(2.0/(self.params['reslimit']*self.freq))) limitwidth = primefactor.getNextEvenPrime(limitwidth) requestres = 2.0/(self.freq*limitwidth) if limitwidth > fftwidth: apDisplay.printError("Cannot get requested resolution" +(" request res %.1fA higher than max res %.1fA for new widths %d > %d" %(requestres, maxres, limitwidth, fftwidth))) apDisplay.printColor("Requested resolution OK: " +(" request res %.1fA less than max res %.1fA with fft widths %d < %d" %(requestres, maxres, limitwidth, fftwidth)), "green") newshape = (limitwidth, limitwidth) fftarray = imagefilter.frame_cut(fftarray, newshape) ### spacing parameters self.mfreq = self.freq*1e10 fftwidth = min(fftarray.shape) self.apix = 1.0/(fftwidth*self.freq) self.ctfvalues['apix'] = self.apix if self.params['sample'] is 'stain': self.ctfvalues['amplitude_contrast'] = 0.14 else: self.ctfvalues['amplitude_contrast'] = 0.07 ### # This is the start of the actual program ### ### this is either simple rotational average, ### or complex elliptical average with edge find and RANSAC raddata, PSDarray = self.from2Dinto1D(fftarray) lowerrad = ctftools.getCtfExtrema(self.params['maxdef'], self.mfreq, self.cs, self.volts, self.ctfvalues['amplitude_contrast'], 1, "valley") lowerbound = numpy.searchsorted(raddata, lowerrad[0]*self.freq) if self.params['sample'] is 'stain': upperbound = numpy.searchsorted(raddata, 1/8.) else: upperbound = numpy.searchsorted(raddata, 1/12.) ### fun way to get initial defocus estimate fitData = self.fitLinearPlus(raddata, PSDarray) #lowessFit = lowess.lowess(raddata**2, PSDarray, smoothing=0.6666) if self.debug is True: from matplotlib import pyplot pyplot.clf() pyplot.plot(raddata**2, PSDarray) pyplot.plot(raddata**2, fitData) pyplot.show() flatPSDarray = PSDarray-fitData flatPSDarray /= numpy.abs(flatPSDarray[lowerbound:upperbound]).max() defocus = self.gridSearch(raddata, flatPSDarray, lowerbound) #defocus = findroots.estimateDefocus(raddata[lowerbound:upperbound], flatPSDarray[lowerbound:upperbound], # cs=self.cs, wavelength=self.wavelength, amp_con=self.ctfvalues['amplitude_contrast'], # mindef=self.params['mindef'], maxdef=self.params['maxdef'], volts=self.volts) amplitudecontrast = sinefit.refineAmplitudeContrast(raddata[lowerbound:upperbound]*1e10, defocus, flatPSDarray[lowerbound:upperbound], self.ctfvalues['cs'], self.wavelength, msg=self.debug) if amplitudecontrast is not None: print "amplitudecontrast", amplitudecontrast self.ctfvalues['amplitude_contrast'] = amplitudecontrast defocus = self.defocusLoop(defocus, raddata, flatPSDarray, lowerbound, upperbound) #lowerrad = ctftools.getCtfExtrema(defocus, self.mfreq, self.cs, self.volts, # self.ctfvalues['amplitude_contrast'], 1, "valley") #lowerbound = numpy.searchsorted(raddata, lowerrad[0]*self.freq) normPSDarray = self.fullTriSectionNormalize(raddata, PSDarray, defocus) defocus = self.defocusLoop(defocus, raddata, normPSDarray, lowerbound, upperbound) self.findEllipseLoop(fftarray, lowerbound, upperbound) self.refineEllipseLoop(fftarray, lowerbound, upperbound) ##================================== ## FINISH UP ##================================== apDisplay.printColor("Finishing up using best found CTF values", "blue") self.printBestValues() if self.bestvalues is None: apDisplay.printColor("No CTF estimate found for this image", "red") self.badprocess = True return ### take best values and use them self.ctfvalues = self.bestvalues self.ellipseParams = self.bestellipse ### stupid fix, get value in millimeters self.ctfvalues['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) ### translate ellipse into ctf values if self.ellipseParams is not None: self.ctfvalues['angle_astigmatism'] = math.degrees(self.ellipseParams['alpha']) ellipratio = self.ellipseParams['a']/self.ellipseParams['b'] phi = math.asin(self.ctfvalues['amplitude_contrast']) #note: a > b then def1 < def2 #major axis self.ctfvalues['defocus1'] = self.ctfvalues['defocus']/ellipratio #minor axis self.ctfvalues['defocus2'] = self.ctfvalues['defocus']*ellipratio defdiff = 1.0 - 2*self.ctfvalues['defocus']/(self.ctfvalues['defocus1']+self.ctfvalues['defocus2']) print "%.3e --> %.3e,%.3e"%(self.ctfvalues['defocus'], self.ctfvalues['defocus2'], self.ctfvalues['defocus1']) print defdiff*100 if defdiff*100 > 1: apDisplay.printWarning("Large astigmatism") #sys.exit(1) else: self.ctfvalues['angle_astigmatism'] = 0.0 self.ctfvalues['defocus1'] = self.ctfvalues['defocus'] self.ctfvalues['defocus2'] = self.ctfvalues['defocus'] if self.ctfvalues['amplitude_contrast'] < 0.0: self.ctfvalues['amplitude_contrast'] = 0.0 if self.ctfvalues['amplitude_contrast'] > self.maxAmpCon: self.ctfvalues['amplitude_contrast'] = self.maxAmpCon if abs(self.ctfvalues['defocus1']) > abs(self.ctfvalues['defocus2']): # incorrect, need to shift angle by 90 degrees apDisplay.printWarning("|def1| > |def2|, flipping defocus axes") tempdef = self.ctfvalues['defocus1'] self.ctfvalues['defocus1'] = self.ctfvalues['defocus2'] self.ctfvalues['defocus2'] = tempdef self.ctfvalues['angle_astigmatism'] += 90 # get astig_angle within range -90 < angle <= 90 while self.ctfvalues['angle_astigmatism'] > 90: self.ctfvalues['angle_astigmatism'] -= 180 while self.ctfvalues['angle_astigmatism'] < -90: self.ctfvalues['angle_astigmatism'] += 180 avgres = self.getResolution(self.ctfvalues['defocus'], raddata, PSDarray, lowerbound) apDisplay.printColor("Final defocus values %.3e -> %.3e, %.3e; ac=%.2f, res=%.1f" %(self.ctfvalues['defocus'], self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['amplitude_contrast'], avgres/2.0), "green") for i in range(10): print "====================================" print "PREVIOUS VALUES" ctfdb.getBestCtfByResolution(imgdata) print "CURRENT VALUES" defocusratio = self.ctfvalues['defocus2']/self.ctfvalues['defocus1'] apDisplay.printColor("def1: %.2e | def2: %.2e | angle: %.1f | ampcontr %.2f | defratio %.3f" %(self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'], self.ctfvalues['amplitude_contrast'], defocusratio), "blue") self.printBestValues() print "====================================" return
def processImage(self, imgdata): self.ctfvalues = {} ### convert image to spider spiderimage = apDisplay.short(imgdata['filename'])+".spi" spider.write(imgdata['image'], spiderimage) ### high tension in kiloVolts kvolts = imgdata['scope']['high tension']/1000.0 ### spherical aberration in millimeters cs = apInstrument.getCsValueFromSession(self.getSessionData()) ### pixel size in Angstroms apix = apDatabase.getPixelSize(imgdata) ### write image name inputstr = "" inputstr += ("\n") inputstr += ("# image filename in spider format\n") inputstr += ("image=%s\n"%(spiderimage)) ### common parameters that never change inputstr += ("\n") inputstr += ("# common parameters that never change\n") inputstr += ("N_horizontal=%d\n"%(self.params['fieldsize'])) #inputstr += ("particle_horizontal=%d\n"%(128)) inputstr += ("show_optimization=yes\n") inputstr += ("micrograph_averaging=yes\n") """ Give a value in digital frequency (i.e. between 0.0 and 0.5) This cut-off prevents the typically peak at the center of the PSD to interfere with CTF estimation. The default value is 0.05, but for micrographs with a very fine sampling this may be lowered towards 0.0 """ #minres = apix/minfreq => minfreq = apix/minres minfreq = apix/self.params['resmin'] inputstr += ("min_freq=%.3f\n"%(minfreq)) """ Give a value in digital frequency (i.e. between 0.0 and 0.5) This cut-off prevents high-resolution terms where only noise exists to interfere with CTF estimation. The default value is 0.35, but it should be increased for micrographs with signals extending beyond this value """ #maxres = apix/maxfreq => maxfreq = apix/maxres maxfreq = apix/self.params['resmax'] inputstr += ("max_freq=%.3f\n"%(maxfreq)) ### CTF input parameters from database inputstr += ("\n") inputstr += ("# CTF input parameters from database\n") inputstr += ("voltage=%d\n"%(kvolts)) inputstr += ("spherical_aberration=%.1f\n"%(cs)) inputstr += ("sampling_rate=%.4f\n"%(apix)) ### best defocus in negative Angstroms ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata, msg=True) if ctfvalue is None: apDisplay.printWarning("Xmipp CTF as implemented in Appion requires an initial CTF estimate to process" +"\nPlease run another CTF program first and try again on this image") self.ctfvalues = None return nominal = (ctfvalue['defocus1']+ctfvalue['defocus2'])/2.0 inputstr += ("defocusU=%d\n"%(-abs(ctfvalue['defocus1'])*1.0e10)) inputstr += ("defocusV=%d\n"%(-abs(ctfvalue['defocus2'])*1.0e10)) inputstr += ("Q0=%d\n"%(-ctfvalue['amplitude_contrast'])) inputstr += ("\n") ### open and write to input parameter file paramfile = apDisplay.short(imgdata['filename'])+"-CTF.prm" f = open(paramfile, "w") print inputstr f.write(inputstr) f.close() #[min_freq=<f=0.05>] #[max_freq=<f=0.35>] xmippcmd = "xmipp_ctf_estimate_from_micrograph -i %s"%(paramfile) #xmippcmd = "echo %s"%(paramfile) t0 = time.time() apDisplay.printMsg("running ctf estimation at "+time.asctime()) proc = subprocess.Popen(xmippcmd, shell=True, stdout=subprocess.PIPE) #waittime = 2 #while proc.poll() is None: # sys.stderr.write(".") # time.sleep(waittime) (stdout, stderr) = proc.communicate() #print (stdout, stderr) apDisplay.printMsg("ctf estimation completed in "+apDisplay.timeString(time.time()-t0)) ### read commandline output to get fit score lines = stdout.split('\n') lastline = "" for line in lines[-50:]: if "--->" in line: lastline = line if not "--->" in lastline: apDisplay.printWarning("Xmipp CTF failed") self.badprocess = True return bits = lastline.split('--->') confidence = float(bits[1]) score = round(math.sqrt(1-confidence), 5) apDisplay.printColor("Confidence value is %.5f (score %.5f)"%(confidence, score), "cyan") f = open("confidence.log", "a") f.write("Confidence value is %.5f for image %s (score %.3f)\n" %(confidence, apDisplay.short(imgdata['filename']), score)) f.close() ### delete image in spider format no longer needed apFile.removeFile(spiderimage) ### read output parameter file outparamfile = apDisplay.short(imgdata['filename'])+"_Periodogramavg.ctfparam" f = open(outparamfile, "r") for line in f: sline = line.strip() bits = sline.split('=') if sline.startswith("defocusU"): defocus1 = float(bits[1].strip()) elif sline.startswith("defocusV"): defocus2 = float(bits[1].strip()) elif sline.startswith("azimuthal_angle"): angle_astigmatism = float(bits[1].strip()) elif sline.startswith("Q0"): amplitude_contrast = abs(float(bits[1].strip())) print defocus1, defocus2, angle_astigmatism, amplitude_contrast #defocusU= -18418.6 #defocusV= -24272.1 #azimuthal_angle= 79.7936 #Q0= -0.346951 #negative of ACE amplitude_contrast print "AMP CONTRAST: %.4f -- %.4f"%(amplitude_contrast, ctfvalue['amplitude_contrast']) self.ctfvalues = { 'defocus1': defocus1*1e-10, 'defocus2': defocus2*1e-10, 'angle_astigmatism': angle_astigmatism, 'amplitude_contrast': amplitude_contrast, 'nominal': nominal, 'defocusinit': nominal, 'confidence_d': score, 'cs': self.params['cs'], 'volts': kvolts*1000.0, } return