def getCtfValueConfidenceForImage(self, imgdata, msg=False): method = self.params['ctfmethod'] ctfrunid = self.params['ctfrunid'] if ctfrunid is None: return ctfdb.getBestCtfValueForImage(imgdata,msg=msg,method=method) else: return ctfdb.getCtfValueForImage(imgdata, ctfrunid, msg=msg, method=method)
def runAceCorrect(imgdict,params): imgname = imgdict['filename'] imgpath = os.path.join(imgdict['session']['image path'], imgname+'.mrc') voltage = (imgdict['scope']['high tension']) apix = apDatabase.getPixelSize(imgdict) ctfvalues, conf = ctfdb.getBestCtfValueForImage(imgdict) ctdimname = imgname ctdimpath = os.path.join(params['rundir'],ctdimname) print "Corrected Image written to " + ctdimpath #pdb.set_trace() acecorrectcommand=("ctfcorrect1('%s', '%s', '%.32f', '%.32f', '%f', '%f', '%f');" % \ (imgpath, ctdimpath, ctfvalues['defocus1'], ctfvalues['defocus2'], -ctfvalues['angle_astigmatism'], voltage, apix)) print acecorrectcommand try: matlab = pymat.open("matlab -nosplash") except: apDisplay.environmentError() raise pymat.eval(matlab, acecorrectcommand) pymat.close(matlab) return
def rejectAceInfo(self, imgdata): shortname = apDisplay.short(imgdata["filename"]) ### get best defocus value ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata) if ctfvalue is None: if self.params["noace"] is True: apDisplay.printColor("\nrejecting no ACE values: " + apDisplay.short(imgdata["filename"]), "yellow") return False else: # apDisplay.printWarning("skipping no ACE values for "+apDisplay.short(imgdata['filename'])) return True ### check that CTF estimation is above confidence threshold if self.params["ctfcutoff"] and conf < self.params["ctfcutoff"]: apDisplay.printColor( "\nrejecting below CTF cutoff: " + apDisplay.short(imgdata["filename"]) + " conf=" + str(round(conf, 3)), "cyan", ) return False ### defocus should be in negative meters if ctfvalue["defocus2"] is not None and ctfvalue["defocus1"] != ctfvalue["defocus2"]: defocus = (ctfvalue["defocus1"] + ctfvalue["defocus2"]) / 2.0 else: defocus = ctfvalue["defocus1"] defocus = -1.0 * abs(defocus) ### skip micrograph that have defocus above or below min & max defocus levels if self.params["mindefocus"] and defocus > self.params["mindefocus"]: apDisplay.printColor( shortname + " defocus (" + str(round(defocus * 1e6, 2)) + " um) is less than mindefocus (" + str(self.params["mindefocus"] * 1e6) + " um)\n", "cyan", ) return False if self.params["maxdefocus"] and defocus < self.params["maxdefocus"]: apDisplay.printColor( shortname + " defocus (" + str(round(defocus * 1e6, 2)) + " um) is greater than maxdefocus (" + str(self.params["maxdefocus"] * 1e6) + " um)\n", "cyan", ) return False return True
def getDefocusAmpConstForImage(self,imgdata,msg=False): ### This function returns defocus defined as negative underfocus method = self.params['ctfmethod'] ctfrunid = self.params['ctfrunid'] if ctfrunid is None: ctfvalue,conf = ctfdb.getBestCtfValueForImage(imgdata,msg=msg,method=method) # make sure we return a negative underfocus value defocus = -(abs(ctfvalue['defocus1'])+abs(ctfvalue['defocus2']))/2 return defocus, ctfvalue['amplitude_contrast'] else: return ctfdb.getDefocusAndampcontrastForImage(imgdata, ctf_estimation_runid=ctfrunid, msg=msg, method=method)
def reprocessImage(self, imgdata): """ Returns True, if an image should be reprocessed False, if an image was processed and should NOT be reprocessed None, if image has not yet been processed e.g. a confidence less than 80% """ if self.params['reprocess'] is None: return None ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata) if ctfvalue is None: return None if conf > self.params['reprocess']: return False else: return True
def rejectAceInfo(self, imgdata): shortname = apDisplay.short(imgdata['filename']) ### get best defocus value ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata) if ctfvalue is None: if self.params['noace'] is True: apDisplay.printColor( "\nrejecting no ACE values: " + apDisplay.short(imgdata['filename']), "yellow") return False else: #apDisplay.printWarning("skipping no ACE values for "+apDisplay.short(imgdata['filename'])) return True ### check that CTF estimation is above confidence threshold if self.params['ctfcutoff'] and conf < self.params['ctfcutoff']: apDisplay.printColor( "\nrejecting below CTF cutoff: " + apDisplay.short(imgdata['filename']) + " conf=" + str(round(conf, 3)), "cyan") return False ### defocus should be in negative meters if ctfvalue['defocus2'] is not None and ctfvalue[ 'defocus1'] != ctfvalue['defocus2']: defocus = (ctfvalue['defocus1'] + ctfvalue['defocus2']) / 2.0 else: defocus = ctfvalue['defocus1'] defocus = -1.0 * abs(defocus) ### skip micrograph that have defocus above or below min & max defocus levels if self.params['mindefocus'] and defocus > self.params['mindefocus']: apDisplay.printColor(shortname+" defocus ("+str(round(defocus*1e6,2))+\ " um) is less than mindefocus ("+str(self.params['mindefocus']*1e6)+" um)\n","cyan") return False if self.params['maxdefocus'] and defocus < self.params['maxdefocus']: apDisplay.printColor(shortname+" defocus ("+str(round(defocus*1e6,2))+\ " um) is greater than maxdefocus ("+str(self.params['maxdefocus']*1e6)+" um)\n","cyan") return False return True
def processImage(self, imgdata): ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata) if ctfvalue is None: return None defocus = ctfvalue['defocus1'] acepath = ctfvalue['acerun']['path']['path'] opimgpath = os.path.join(acepath,"opimages") #print opimgpath #for i,v in ctfvalue.items(): # print i,v if conf < 0.5: confdir = "doubtful" elif conf > 0.8: confdir = "probable" else: confdir = "questionable" defocusdir = "defocus"+str(int(math.floor(defocus*-1e6))) apParam.createDirectory(os.path.join(self.params['opimage1'], defocusdir), warning=False) apParam.createDirectory(os.path.join(self.params['opimage2'], defocusdir), warning=False) #copy first file opfile = os.path.join(opimgpath, ctfvalue['graph1']) if os.path.isfile(opfile): shutil.copyfile(opfile, os.path.join(self.params['opimage1'], "all", ctfvalue['graph1'])) shutil.copyfile(opfile, os.path.join(self.params['opimage1'], confdir, ctfvalue['graph1'])) shutil.copyfile(opfile, os.path.join(self.params['opimage1'], defocusdir, ctfvalue['graph1'])) else: apDisplay.printWarning("could not find opimage: "+opfile) #copy second file opfile = os.path.join(opimgpath, ctfvalue['graph2']) if os.path.isfile(opfile): shutil.copyfile(opfile, os.path.join(self.params['opimage2'], "all", ctfvalue['graph2'])) shutil.copyfile(opfile, os.path.join(self.params['opimage2'], confdir, ctfvalue['graph2'])) shutil.copyfile(opfile, os.path.join(self.params['opimage2'], defocusdir, ctfvalue['graph2'])) else: apDisplay.printWarning("could not find opimage: "+opfile)
def start(self): ### default parameters starfile = self.params['starfile'] images = self.params['images'] dbnum = self.params['dbnum'] ctfrunid = self.params['ctfrunid'] stackid = self.params['stackid'] voltage = self.params['voltage'] cs = self.params['cs'] wgh = self.params['wgh'] ### particles, angles appiondata.sinedon.setConfig('appiondata', db="ap%d" % dbnum) particledata = apStack.getStackParticlesFromId(stackid) ### write Relion starfile header sf = open(starfile, "w") sf.write("data_images\n") sf.write("loop_\n") sf.write("_rlnImageName\n") sf.write("_rlnMicrographName\n") sf.write("_rlnDefocusU\n") sf.write("_rlnDefocusV\n") sf.write("_rlnDefocusAngle\n") sf.write("_rlnVoltage\n") sf.write("_rlnSphericalAberration\n") sf.write("_rlnAmplitudeContrast\n") ### write info to starfile olddx = 0 micn = 0 oldimgid = None for i in range(len(particledata)): if i % 1000 == 0: print "done with %d particles" % i ### CTF info image = particledata[i]['particle']['image'] imgid = image.dbid try: ctf = ctfdb.getCtfValueForCtfRunId(image, ctfrunid, msg=False) dx = ctf['defocus1'] * 10e9 dy = ctf['defocus2'] * 10e9 astig = ctf['angle_astigmatism'] except: ctf = None # print "didn't find CTF values for image ", image.dbid if ctf is None: if oldimgid != imgid: print "particle %d: " % i, "getting best value for image: %d" % imgid ctf = ctfdb.getBestCtfValueForImage(image, msg=False, method='ctffind') dx = ctf[0]['defocus1'] * 10e9 dy = ctf[0]['defocus2'] * 10e9 astig = ctf[0]['angle_astigmatism'] oldctf = ctf oldimgid = imgid else: try: ctf = oldctf dx = oldctf[0]['defocus1'] * 10e9 dy = oldctf[0]['defocus2'] * 10e9 astig = oldctf[0]['angle_astigmatism'] except: apDisplay.printError("no CTF information for image") if dx != olddx: micn += 1 olddx = dx ### write input Relion parameters sf.write("%06d@%s%10d%12.3f%12.3f%12.3f%8.3f%8.3f%8.3f\n" % (i + 1, images, micn, dx, dy, astig, voltage, cs, wgh)) sf.close()
def generateParticleParams(self): paramfile = os.path.join(self.params['rundir'], 'params.iter000.par') apDisplay.printMsg("Creating parameter file: "+paramfile) numpart = apStack.getNumberStackParticlesFromId(self.params['stackid']) if os.path.isfile(paramfile): f = open(paramfile, 'r') numparam = len(f.readlines()) if numparam > self.params['last']: apDisplay.printMsg("Param file exists") return paramfile else: apDisplay.printWarning("Param file exists with too few particles: %d vs %d"%(numparam,numpart)) stackdata = apStack.getStackParticlesFromId(self.params['stackid']) particleparams={} f = open(paramfile, 'w') f.write("C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n") apDisplay.printMsg("Writing out particle parameters") count = 0 t0 = time.time() self.noeulers = 0 for particle in stackdata: count += 1 if (count % 200 == 0): estime = (time.time() - t0) * (numpart-count) / float(count) apDisplay.printMsg("particle %d -- %s remain"%(count, apDisplay.timeString(estime))) if count > self.params['last']: break # defaults ## Niko says that if the defocus is negative it will not perform CTF correction ## But it will also not correct the amplitudes particleparams = { 'ptclnum': particle['particleNumber'], 'psi': 0.0, 'theta': 0.0, 'phi': 0.0, 'df1': -1.0, # workaround if no ctf correction 'df2': -1.0, # set defocus to -1.0 Angstroms 'angast': 0.0, 'mag': 10000, # workaround to get around dstep 'shx': 0.0, 'shy': 0.0, 'film': 1, 'presa': 0.0, 'dpres': 0.0, } imagedata = particle['particle']['image'] if self.params['noctf'] is False: ctfdata, confidence = ctfdb.getBestCtfValueForImage(imagedata, msg=False, method=self.params['ctfmethod']) if ctfdata is not None: ### use defocus and astigmatism values particleparams['df1'] = abs(ctfdata['defocus1']*1e10) particleparams['df2'] = abs(ctfdata['defocus2']*1e10) particleparams['angast'] = ctfdata['angle_astigmatism'] # if using parameters from previous reconstruction if self.params['reconiterid'] is not None: emaneuler = self.getStackParticleEulersForIteration(particle['particleNumber']) frealigneulers = apFrealign.convertEmanEulersToFrealign(emaneuler, sym=self.params['sym']) particleparams['psi'] = frealigneulers['psi'] particleparams['theta'] = frealigneulers['theta'] particleparams['phi'] = frealigneulers['phi'] particleparams['shx'] = emaneuler['shiftx'] particleparams['shy'] = emaneuler['shifty'] if emaneuler['mirror'] is True: particleparams['shx'] *= -1 self.writeParticleParamLine(particleparams,f) f.close() return paramfile
def processImage(self, imgdata): self.ctfvalues = {} ### convert image to spider spiderimage = apDisplay.short(imgdata['filename']) + ".spi" spider.write(imgdata['image'], spiderimage) ### high tension in kiloVolts kvolts = imgdata['scope']['high tension'] / 1000.0 ### spherical aberration in millimeters cs = apInstrument.getCsValueFromSession(self.getSessionData()) ### pixel size in Angstroms apix = apDatabase.getPixelSize(imgdata) ### write image name inputstr = "" inputstr += ("\n") inputstr += ("# image filename in spider format\n") inputstr += ("image=%s\n" % (spiderimage)) ### common parameters that never change inputstr += ("\n") inputstr += ("# common parameters that never change\n") inputstr += ("N_horizontal=%d\n" % (self.params['fieldsize'])) #inputstr += ("particle_horizontal=%d\n"%(128)) inputstr += ("show_optimization=yes\n") inputstr += ("micrograph_averaging=yes\n") """ Give a value in digital frequency (i.e. between 0.0 and 0.5) This cut-off prevents the typically peak at the center of the PSD to interfere with CTF estimation. The default value is 0.05, but for micrographs with a very fine sampling this may be lowered towards 0.0 """ #minres = apix/minfreq => minfreq = apix/minres minfreq = apix / self.params['resmin'] inputstr += ("min_freq=%.3f\n" % (minfreq)) """ Give a value in digital frequency (i.e. between 0.0 and 0.5) This cut-off prevents high-resolution terms where only noise exists to interfere with CTF estimation. The default value is 0.35, but it should be increased for micrographs with signals extending beyond this value """ #maxres = apix/maxfreq => maxfreq = apix/maxres maxfreq = apix / self.params['resmax'] inputstr += ("max_freq=%.3f\n" % (maxfreq)) ### CTF input parameters from database inputstr += ("\n") inputstr += ("# CTF input parameters from database\n") inputstr += ("voltage=%d\n" % (kvolts)) inputstr += ("spherical_aberration=%.1f\n" % (cs)) inputstr += ("sampling_rate=%.4f\n" % (apix)) ### best defocus in negative Angstroms ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata, msg=True) nominal = (ctfvalue['defocus1'] + ctfvalue['defocus2']) / 2.0 inputstr += ("defocusU=%d\n" % (-abs(ctfvalue['defocus1']) * 1.0e10)) inputstr += ("defocusV=%d\n" % (-abs(ctfvalue['defocus2']) * 1.0e10)) inputstr += ("Q0=%d\n" % (-ctfvalue['amplitude_contrast'])) inputstr += ("\n") ### open and write to input parameter file paramfile = apDisplay.short(imgdata['filename']) + "-CTF.prm" f = open(paramfile, "w") print inputstr f.write(inputstr) f.close() #[min_freq=<f=0.05>] #[max_freq=<f=0.35>] xmippcmd = "xmipp_ctf_estimate_from_micrograph -i %s" % (paramfile) #xmippcmd = "echo %s"%(paramfile) t0 = time.time() apDisplay.printMsg("running ctf estimation at " + time.asctime()) proc = subprocess.Popen(xmippcmd, shell=True, stdout=subprocess.PIPE) #waittime = 2 #while proc.poll() is None: # sys.stderr.write(".") # time.sleep(waittime) (stdout, stderr) = proc.communicate() #print (stdout, stderr) apDisplay.printMsg("ctf estimation completed in " + apDisplay.timeString(time.time() - t0)) ### read commandline output to get fit score lines = stdout.split('\n') lastline = "" for line in lines[-50:]: if "--->" in line: lastline = line if not "--->" in lastline: apDisplay.printWarning("Xmipp CTF failed") self.badprocess = True return bits = lastline.split('--->') confidence = float(bits[1]) score = round(math.sqrt(1 - confidence), 5) apDisplay.printColor( "Confidence value is %.5f (score %.5f)" % (confidence, score), "cyan") f = open("confidence.log", "a") f.write("Confidence value is %.5f for image %s (score %.3f)\n" % (confidence, apDisplay.short(imgdata['filename']), score)) f.close() ### delete image in spider format no longer needed apFile.removeFile(spiderimage) ### read output parameter file outparamfile = apDisplay.short( imgdata['filename']) + "_Periodogramavg.ctfparam" f = open(outparamfile, "r") for line in f: sline = line.strip() bits = sline.split('=') if sline.startswith("defocusU"): defocus1 = float(bits[1].strip()) elif sline.startswith("defocusV"): defocus2 = float(bits[1].strip()) elif sline.startswith("azimuthal_angle"): angle_astigmatism = float(bits[1].strip()) elif sline.startswith("Q0"): amplitude_contrast = abs(float(bits[1].strip())) print defocus1, defocus2, angle_astigmatism, amplitude_contrast #defocusU= -18418.6 #defocusV= -24272.1 #azimuthal_angle= 79.7936 #Q0= -0.346951 #negative of ACE amplitude_contrast print "AMP CONTRAST: %.4f -- %.4f" % (amplitude_contrast, ctfvalue['amplitude_contrast']) self.ctfvalues = { 'defocus1': defocus1 * 1e-10, 'defocus2': defocus2 * 1e-10, 'angle_astigmatism': angle_astigmatism, 'amplitude_contrast': amplitude_contrast, 'nominal': nominal, 'defocusinit': nominal, 'confidence_d': score, 'cs': self.params['cs'], 'volts': kvolts * 1000.0, } return
def start(self): ### default parameters starfile = self.params['starfile'] images = self.params['images'] dbnum = self.params['dbnum'] ctfrunid = self.params['ctfrunid'] stackid = self.params['stackid'] voltage = self.params['voltage'] cs = self.params['cs'] wgh = self.params['wgh'] ### particles, angles appiondata.sinedon.setConfig('appiondata', db="ap%d" % dbnum) particledata = apStack.getStackParticlesFromId(stackid) ### write Relion starfile header sf = open(starfile, "w") sf.write("data_images\n") sf.write("loop_\n") sf.write("_rlnImageName\n") sf.write("_rlnMicrographName\n") sf.write("_rlnDefocusU\n") sf.write("_rlnDefocusV\n") sf.write("_rlnDefocusAngle\n") sf.write("_rlnVoltage\n") sf.write("_rlnSphericalAberration\n") sf.write("_rlnAmplitudeContrast\n") ### write info to starfile olddx = 0 micn = 0 oldimgid = None for i in range(len(particledata)): if i % 1000 == 0: print "done with %d particles" % i ### CTF info image = particledata[i]['particle']['image'] imgid = image.dbid try: ctf = ctfdb.getCtfValueForCtfRunId(image, ctfrunid, msg=False) dx = ctf['defocus1'] * 10e9 dy = ctf['defocus2'] * 10e9 astig = ctf['angle_astigmatism'] except: ctf = None # print "didn't find CTF values for image ", image.dbid if ctf is None: if oldimgid != imgid: print "particle %d: " % i, "getting best value for image: %d" % imgid ctf = ctfdb.getBestCtfValueForImage(image, msg=False, method='ctffind') dx = ctf[0]['defocus1'] * 10e9 dy = ctf[0]['defocus2'] * 10e9 astig = ctf[0]['angle_astigmatism'] oldctf = ctf oldimgid = imgid else: try: ctf = oldctf dx = oldctf[0]['defocus1'] * 10e9 dy = oldctf[0]['defocus2'] * 10e9 astig = oldctf[0]['angle_astigmatism'] except: apDisplay.printError("no CTF information for image") if dx != olddx: micn += 1 olddx = dx ### write input Relion parameters sf.write("%06d@%s%10d%12.3f%12.3f%12.3f%8.3f%8.3f%8.3f\n" % (i+1, images, micn, dx, dy, astig, voltage, cs, wgh) ) sf.close()
def processImage(self, imgdata): ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata) if ctfvalue is None: return None defocus = ctfvalue['defocus1'] acepath = ctfvalue['acerun']['path']['path'] opimgpath = os.path.join(acepath, "opimages") #print opimgpath #for i,v in ctfvalue.items(): # print i,v if conf < 0.5: confdir = "doubtful" elif conf > 0.8: confdir = "probable" else: confdir = "questionable" defocusdir = "defocus" + str(int(math.floor(defocus * -1e6))) apParam.createDirectory(os.path.join(self.params['opimage1'], defocusdir), warning=False) apParam.createDirectory(os.path.join(self.params['opimage2'], defocusdir), warning=False) #copy first file opfile = os.path.join(opimgpath, ctfvalue['graph1']) if os.path.isfile(opfile): shutil.copyfile( opfile, os.path.join(self.params['opimage1'], "all", ctfvalue['graph1'])) shutil.copyfile( opfile, os.path.join(self.params['opimage1'], confdir, ctfvalue['graph1'])) shutil.copyfile( opfile, os.path.join(self.params['opimage1'], defocusdir, ctfvalue['graph1'])) else: apDisplay.printWarning("could not find opimage: " + opfile) #copy second file opfile = os.path.join(opimgpath, ctfvalue['graph2']) if os.path.isfile(opfile): shutil.copyfile( opfile, os.path.join(self.params['opimage2'], "all", ctfvalue['graph2'])) shutil.copyfile( opfile, os.path.join(self.params['opimage2'], confdir, ctfvalue['graph2'])) shutil.copyfile( opfile, os.path.join(self.params['opimage2'], defocusdir, ctfvalue['graph2'])) else: apDisplay.printWarning("could not find opimage: " + opfile)
def generateParticleParams(self): paramfile = os.path.join(self.params['rundir'], 'params.iter000.par') apDisplay.printMsg("Creating parameter file: " + paramfile) numpart = apStack.getNumberStackParticlesFromId(self.params['stackid']) if os.path.isfile(paramfile): f = open(paramfile, 'r') numparam = len(f.readlines()) if numparam > self.params['last']: apDisplay.printMsg("Param file exists") return paramfile else: apDisplay.printWarning( "Param file exists with too few particles: %d vs %d" % (numparam, numpart)) stackdata = apStack.getStackParticlesFromId(self.params['stackid']) particleparams = {} f = open(paramfile, 'w') f.write( "C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n" ) apDisplay.printMsg("Writing out particle parameters") count = 0 t0 = time.time() self.noeulers = 0 for particle in stackdata: count += 1 if (count % 200 == 0): estime = (time.time() - t0) * (numpart - count) / float(count) apDisplay.printMsg("particle %d -- %s remain" % (count, apDisplay.timeString(estime))) if count > self.params['last']: break # defaults ## Niko says that if the defocus is negative it will not perform CTF correction ## But it will also not correct the amplitudes particleparams = { 'ptclnum': particle['particleNumber'], 'psi': 0.0, 'theta': 0.0, 'phi': 0.0, 'df1': -1.0, # workaround if no ctf correction 'df2': -1.0, # set defocus to -1.0 Angstroms 'angast': 0.0, 'mag': 10000, # workaround to get around dstep 'shx': 0.0, 'shy': 0.0, 'film': 1, 'presa': 0.0, 'dpres': 0.0, } imagedata = particle['particle']['image'] if self.params['noctf'] is False: ctfdata, confidence = ctfdb.getBestCtfValueForImage( imagedata, msg=False, method=self.params['ctfmethod']) if ctfdata is not None: ### use defocus and astigmatism values particleparams['df1'] = abs(ctfdata['defocus1'] * 1e10) particleparams['df2'] = abs(ctfdata['defocus2'] * 1e10) particleparams['angast'] = ctfdata['angle_astigmatism'] # if using parameters from previous reconstruction if self.params['reconiterid'] is not None: emaneuler = self.getStackParticleEulersForIteration( particle['particleNumber']) frealigneulers = apFrealign.convertEmanEulersToFrealign( emaneuler, sym=self.params['sym']) particleparams['psi'] = frealigneulers['psi'] particleparams['theta'] = frealigneulers['theta'] particleparams['phi'] = frealigneulers['phi'] particleparams['shx'] = emaneuler['shiftx'] particleparams['shy'] = emaneuler['shifty'] if emaneuler['mirror'] is True: particleparams['shx'] *= -1 self.writeParticleParamLine(particleparams, f) f.close() return paramfile
def generateParticleParams(params, modeldata, initparfile='params.0.par'): params['inpar'] = os.path.join(params['rundir'], initparfile) apDisplay.printMsg("Creating parameter file: " + params['inpar']) params['mode'] = 3 stackdata = getStackParticlesInOrder(params) first_imageid = stackdata[0]['particle']['image'].dbid f = open(params['inpar'], 'w') params['noClassification'] = 0 if params['reconiterid']: iterdata = apRecon.getRefineIterDataFromIterationId( params['reconiterid']) sym_name = modeldata['symmetry']['symmetry'] print "Writing out particle parameters" if 'last' not in params: params['last'] = len(stackdata) for i, particle in enumerate(stackdata[:params['last']]): # defaults particleparams = initializeParticleParams(i) # for helical reconstructions, film is helix number if particle['particle']['helixnum']: imgid = particle['particle']['image'].dbid helix = particle['particle']['helixnum'] try: if params['lastimgid'] != imgid or params['lasthelix'] != helix: params['totalHelix'] += 1 except: params['totalHelix'] = 1 params['lastimgid'] = imgid params['lasthelix'] = helix particleparams['film'] = params['totalHelix'] else: particleparams['film'] = particle['particle'][ 'image'].dbid - first_imageid + 1 # extra particle number information not read by Frealign particleparams['pnumber'] = particle['particleNumber'] imagedata = particle['particle']['image'] if params['noctf'] is False: if params['defocpair'] is True: imagedata = apDefocalPairs.getDefocusPair(imagedata) # get tilted parameters first: if params['ctftilt'] is True: ctfdata = ctfdb.getBestTiltCtfValueForImage(imagedata) if ctfdata is None: apDisplay.printError("Failed to get ctf parameters") # get x,y coordinate of the particle nx = particle['particle']['xcoord'] ny = particle['particle']['ycoord'] df1, df2 = ctfdb.getParticleTiltDefocus( ctfdata, imagedata, nx, ny) # use defocus & astigmatism values particleparams['df1'] = abs(df1) particleparams['df2'] = abs(df2) particleparams['angast'] = ctfdata['angle_astigmatism'] else: # first see if there are ctf values ctfdata, confidence = ctfdb.getBestCtfValueForImage( imagedata, msg=False, method=params['ctfmethod']) ## Do we really want value from any method if it is specified? if ctfdata is None: ctfdata, confidence = ctfdb.getBestCtfValueForImage( imagedata, msg=False) if ctfdata is not None: # use defocus & astigmatism values particleparams['df1'] = abs(ctfdata['defocus1'] * 1e10) particleparams['df2'] = abs(ctfdata['defocus2'] * 1e10) particleparams['angast'] = ctfdata['angle_astigmatism'] # if using parameters from previous reconstruction if params['reconiterid'] is not None: params['mode'] = 1 getStackParticleEulersForIteration(params, particle['particleNumber']) fr_eulers = convertAppionEmanEulersToFrealign( params['eman_orient'], sym_name) particleparams['psi'] = fr_eulers['psi'] particleparams['theta'] = fr_eulers['theta'] particleparams['phi'] = fr_eulers['phi'] particleparams['shx'] = -params['eman_orient']['shiftx'] particleparams['shy'] = -params['eman_orient']['shifty'] if params['eman_orient']['mirror'] is True: particleparams['shx'] *= -1 writeParticleParamLine(particleparams, f) f.close()
def runAce(matlab, imgdata, params, showprev=True): imgname = imgdata['filename'] if showprev is True: bestctfvalue, bestconf = ctfdb.getBestCtfValueForImage(imgdata) if bestctfvalue: print ( "Prev best: '"+bestctfvalue['acerun']['name']+"', conf="+ apDisplay.colorProb(bestconf)+", defocus="+str(round(-1.0*abs(bestctfvalue['defocus1']*1.0e6),2))+ " microns" ) if params['uncorrected']: tmpname='temporaryCorrectedImage.mrc' imgarray = apDBImage.correctImage(imgdata) imgpath = os.path.join(params['rundir'],tmpname) apImage.arrayToMrc(imgarray, imgpath) print "processing", imgpath else: imgpath = os.path.join(imgdata['session']['image path'], imgname+'.mrc') nominal = None if params['nominal'] is not None: nominal=params['nominal'] elif params['newnominal'] is True: nominal = ctfdb.getBestDefocusForImage(imgdata, msg=True) if nominal is None: nominal = imgdata['scope']['defocus'] if nominal is None or nominal > 0 or nominal < -15e-6: apDisplay.printWarning("Nominal should be of the form nominal=-1.2e-6"+\ " for -1.2 microns NOT:"+str(nominal)) #Neil's Hack #if 'autosample' in params and params['autosample']: # x = abs(nominal*1.0e6) # val = 1.585 + 0.057587 * x - 0.044106 * x**2 + 0.010877 * x**3 # resamplefr_override = round(val,3) # print "resamplefr_override=",resamplefr_override # pymat.eval(matlab, "resamplefr="+str(resamplefr_override)+";") pymat.eval(matlab,("dforig = %e;" % nominal)) if params['stig'] == 0: plist = (imgpath, params['outtextfile'], params['display'], params['stig'],\ params['medium'], -nominal, params['tempdir']+"/") acecmd = makeMatlabCmd("ctfparams = ace(",");",plist) else: plist = (imgname, imgpath, params['outtextfile'], params['opimagedir'], \ params['matdir'], params['display'], params['stig'],\ params['medium'], -nominal, params['tempdir']+"/", params['resamplefr']) acecmd = makeMatlabCmd("ctfparams = measureAstigmatism(",");",plist) #print acecmd pymat.eval(matlab,acecmd) matfile = os.path.join(params['matdir'], imgname+".mrc.mat") if params['stig']==0: savematcmd = "save('"+matfile+"','ctfparams','scopeparams', 'dforig');" pymat.eval(matlab,savematcmd) ctfvalue = pymat.get(matlab, 'ctfparams') ctfvalue=ctfvalue[0] ctfdb.printResults(params, nominal, ctfvalue) return ctfvalue
def runAce(matlab, imgdata, params, showprev=True): imgname = imgdata['filename'] if showprev is True: bestctfvalue, bestconf = ctfdb.getBestCtfValueForImage(imgdata) if bestctfvalue: print("Prev best: '" + bestctfvalue['acerun']['name'] + "', conf=" + apDisplay.colorProb(bestconf) + ", defocus=" + str(round(-1.0 * abs(bestctfvalue['defocus1'] * 1.0e6), 2)) + " microns") if params['uncorrected']: tmpname = 'temporaryCorrectedImage.mrc' imgarray = apDBImage.correctImage(imgdata) imgpath = os.path.join(params['rundir'], tmpname) apImage.arrayToMrc(imgarray, imgpath) print "processing", imgpath else: imgpath = os.path.join(imgdata['session']['image path'], imgname + '.mrc') nominal = None if params['nominal'] is not None: nominal = params['nominal'] elif params['newnominal'] is True: nominal = ctfdb.getBestDefocusForImage(imgdata, msg=True) if nominal is None: nominal = imgdata['scope']['defocus'] if nominal is None or nominal > 0 or nominal < -15e-6: apDisplay.printWarning("Nominal should be of the form nominal=-1.2e-6"+\ " for -1.2 microns NOT:"+str(nominal)) #Neil's Hack #if 'autosample' in params and params['autosample']: # x = abs(nominal*1.0e6) # val = 1.585 + 0.057587 * x - 0.044106 * x**2 + 0.010877 * x**3 # resamplefr_override = round(val,3) # print "resamplefr_override=",resamplefr_override # pymat.eval(matlab, "resamplefr="+str(resamplefr_override)+";") pymat.eval(matlab, ("dforig = %e;" % nominal)) if params['stig'] == 0: plist = (imgpath, params['outtextfile'], params['display'], params['stig'],\ params['medium'], -nominal, params['tempdir']+"/") acecmd = makeMatlabCmd("ctfparams = ace(", ");", plist) else: plist = (imgname, imgpath, params['outtextfile'], params['opimagedir'], \ params['matdir'], params['display'], params['stig'],\ params['medium'], -nominal, params['tempdir']+"/", params['resamplefr']) acecmd = makeMatlabCmd("ctfparams = measureAstigmatism(", ");", plist) #print acecmd pymat.eval(matlab, acecmd) matfile = os.path.join(params['matdir'], imgname + ".mrc.mat") if params['stig'] == 0: savematcmd = "save('" + matfile + "','ctfparams','scopeparams', 'dforig');" pymat.eval(matlab, savematcmd) ctfvalue = pymat.get(matlab, 'ctfparams') ctfvalue = ctfvalue[0] ctfdb.printResults(params, nominal, ctfvalue) return ctfvalue
def generateParticleParams(params,modeldata,initparfile='params.0.par'): params['inpar']=os.path.join(params['rundir'],initparfile) apDisplay.printMsg("Creating parameter file: "+params['inpar']) params['mode']=3 stackdata=getStackParticlesInOrder(params) first_imageid = stackdata[0]['particle']['image'].dbid f=open(params['inpar'],'w') params['noClassification']=0 if params['reconiterid']: iterdata = apRecon.getRefineIterDataFromIterationId(params['reconiterid']) sym_name = modeldata['symmetry']['symmetry'] print "Writing out particle parameters" if 'last' not in params: params['last'] = len(stackdata) for i, particle in enumerate(stackdata[:params['last']]): # defaults particleparams = initializeParticleParams(i) # for helical reconstructions, film is helix number if particle['particle']['helixnum']: imgid=particle['particle']['image'].dbid helix=particle['particle']['helixnum'] try: if params['lastimgid']!=imgid or params['lasthelix']!=helix: params['totalHelix']+=1 except: params['totalHelix']=1 params['lastimgid']=imgid params['lasthelix']=helix particleparams['film']=params['totalHelix'] else: particleparams['film']=particle['particle']['image'].dbid - first_imageid + 1 # extra particle number information not read by Frealign particleparams['pnumber']=particle['particleNumber'] imagedata=particle['particle']['image'] if params['noctf'] is False: if params['defocpair'] is True: imagedata = apDefocalPairs.getDefocusPair(imagedata) # get tilted parameters first: if params['ctftilt'] is True: ctfdata = ctfdb.getBestTiltCtfValueForImage(imagedata) if ctfdata is None: apDisplay.printError("Failed to get ctf parameters") # get x,y coordinate of the particle nx = particle['particle']['xcoord'] ny = particle['particle']['ycoord'] df1,df2 = ctfdb.getParticleTiltDefocus(ctfdata,imagedata,nx,ny) # use defocus & astigmatism values particleparams['df1']=abs(df1) particleparams['df2']=abs(df2) particleparams['angast']=ctfdata['angle_astigmatism'] else: # first see if there are ctf values ctfdata, confidence=ctfdb.getBestCtfValueForImage(imagedata, msg=False,method=params['ctfmethod']) ## Do we really want value from any method if it is specified? if ctfdata is None: ctfdata, confidence=ctfdb.getBestCtfValueForImage(imagedata, msg=False) if ctfdata is not None: # use defocus & astigmatism values particleparams['df1']=abs(ctfdata['defocus1']*1e10) particleparams['df2']=abs(ctfdata['defocus2']*1e10) particleparams['angast']=ctfdata['angle_astigmatism'] # if using parameters from previous reconstruction if params['reconiterid'] is not None: params['mode']=1 getStackParticleEulersForIteration(params,particle['particleNumber']) fr_eulers = convertAppionEmanEulersToFrealign(params['eman_orient'],sym_name) particleparams['psi'] = fr_eulers['psi'] particleparams['theta'] = fr_eulers['theta'] particleparams['phi'] = fr_eulers['phi'] particleparams['shx']=-params['eman_orient']['shiftx'] particleparams['shy']=-params['eman_orient']['shifty'] if params['eman_orient']['mirror'] is True: particleparams['shx']*=-1 writeParticleParamLine(particleparams,f) f.close()
def processImage(self, imgdata): self.ctfvalues = {} ### convert image to spider spiderimage = apDisplay.short(imgdata['filename'])+".spi" spider.write(imgdata['image'], spiderimage) ### high tension in kiloVolts kvolts = imgdata['scope']['high tension']/1000.0 ### spherical aberration in millimeters cs = apInstrument.getCsValueFromSession(self.getSessionData()) ### pixel size in Angstroms apix = apDatabase.getPixelSize(imgdata) ### write image name inputstr = "" inputstr += ("\n") inputstr += ("# image filename in spider format\n") inputstr += ("image=%s\n"%(spiderimage)) ### common parameters that never change inputstr += ("\n") inputstr += ("# common parameters that never change\n") inputstr += ("N_horizontal=%d\n"%(self.params['fieldsize'])) #inputstr += ("particle_horizontal=%d\n"%(128)) inputstr += ("show_optimization=yes\n") inputstr += ("micrograph_averaging=yes\n") """ Give a value in digital frequency (i.e. between 0.0 and 0.5) This cut-off prevents the typically peak at the center of the PSD to interfere with CTF estimation. The default value is 0.05, but for micrographs with a very fine sampling this may be lowered towards 0.0 """ #minres = apix/minfreq => minfreq = apix/minres minfreq = apix/self.params['resmin'] inputstr += ("min_freq=%.3f\n"%(minfreq)) """ Give a value in digital frequency (i.e. between 0.0 and 0.5) This cut-off prevents high-resolution terms where only noise exists to interfere with CTF estimation. The default value is 0.35, but it should be increased for micrographs with signals extending beyond this value """ #maxres = apix/maxfreq => maxfreq = apix/maxres maxfreq = apix/self.params['resmax'] inputstr += ("max_freq=%.3f\n"%(maxfreq)) ### CTF input parameters from database inputstr += ("\n") inputstr += ("# CTF input parameters from database\n") inputstr += ("voltage=%d\n"%(kvolts)) inputstr += ("spherical_aberration=%.1f\n"%(cs)) inputstr += ("sampling_rate=%.4f\n"%(apix)) ### best defocus in negative Angstroms ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata, msg=True) if ctfvalue is None: apDisplay.printWarning("Xmipp CTF as implemented in Appion requires an initial CTF estimate to process" +"\nPlease run another CTF program first and try again on this image") self.ctfvalues = None return nominal = (ctfvalue['defocus1']+ctfvalue['defocus2'])/2.0 inputstr += ("defocusU=%d\n"%(-abs(ctfvalue['defocus1'])*1.0e10)) inputstr += ("defocusV=%d\n"%(-abs(ctfvalue['defocus2'])*1.0e10)) inputstr += ("Q0=%d\n"%(-ctfvalue['amplitude_contrast'])) inputstr += ("\n") ### open and write to input parameter file paramfile = apDisplay.short(imgdata['filename'])+"-CTF.prm" f = open(paramfile, "w") print inputstr f.write(inputstr) f.close() #[min_freq=<f=0.05>] #[max_freq=<f=0.35>] xmippcmd = "xmipp_ctf_estimate_from_micrograph -i %s"%(paramfile) #xmippcmd = "echo %s"%(paramfile) t0 = time.time() apDisplay.printMsg("running ctf estimation at "+time.asctime()) proc = subprocess.Popen(xmippcmd, shell=True, stdout=subprocess.PIPE) #waittime = 2 #while proc.poll() is None: # sys.stderr.write(".") # time.sleep(waittime) (stdout, stderr) = proc.communicate() #print (stdout, stderr) apDisplay.printMsg("ctf estimation completed in "+apDisplay.timeString(time.time()-t0)) ### read commandline output to get fit score lines = stdout.split('\n') lastline = "" for line in lines[-50:]: if "--->" in line: lastline = line if not "--->" in lastline: apDisplay.printWarning("Xmipp CTF failed") self.badprocess = True return bits = lastline.split('--->') confidence = float(bits[1]) score = round(math.sqrt(1-confidence), 5) apDisplay.printColor("Confidence value is %.5f (score %.5f)"%(confidence, score), "cyan") f = open("confidence.log", "a") f.write("Confidence value is %.5f for image %s (score %.3f)\n" %(confidence, apDisplay.short(imgdata['filename']), score)) f.close() ### delete image in spider format no longer needed apFile.removeFile(spiderimage) ### read output parameter file outparamfile = apDisplay.short(imgdata['filename'])+"_Periodogramavg.ctfparam" f = open(outparamfile, "r") for line in f: sline = line.strip() bits = sline.split('=') if sline.startswith("defocusU"): defocus1 = float(bits[1].strip()) elif sline.startswith("defocusV"): defocus2 = float(bits[1].strip()) elif sline.startswith("azimuthal_angle"): angle_astigmatism = float(bits[1].strip()) elif sline.startswith("Q0"): amplitude_contrast = abs(float(bits[1].strip())) print defocus1, defocus2, angle_astigmatism, amplitude_contrast #defocusU= -18418.6 #defocusV= -24272.1 #azimuthal_angle= 79.7936 #Q0= -0.346951 #negative of ACE amplitude_contrast print "AMP CONTRAST: %.4f -- %.4f"%(amplitude_contrast, ctfvalue['amplitude_contrast']) self.ctfvalues = { 'defocus1': defocus1*1e-10, 'defocus2': defocus2*1e-10, 'angle_astigmatism': angle_astigmatism, 'amplitude_contrast': amplitude_contrast, 'nominal': nominal, 'defocusinit': nominal, 'confidence_d': score, 'cs': self.params['cs'], 'volts': kvolts*1000.0, } return
def generateParticleParams(self): paramfile = os.path.join(self.params["rundir"], "params.iter000.par") apDisplay.printMsg("Creating parameter file: " + paramfile) numpart = apStack.getNumberStackParticlesFromId(self.params["stackid"]) if os.path.isfile(paramfile): f = open(paramfile, "r") numparam = len(f.readlines()) if numparam > self.params["last"]: apDisplay.printMsg("Param file exists") return paramfile else: apDisplay.printWarning("Param file exists with too few particles: %d vs %d" % (numparam, numpart)) stackdata = apStack.getStackParticlesFromId(self.params["stackid"]) particleparams = {} f = open(paramfile, "w") f.write("C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n") apDisplay.printMsg("Writing out particle parameters") count = 0 t0 = time.time() self.noeulers = 0 for particle in stackdata: count += 1 if count % 200 == 0: estime = (time.time() - t0) * (numpart - count) / float(count) apDisplay.printMsg("particle %d -- %s remain" % (count, apDisplay.timeString(estime))) if count > self.params["last"]: break # defaults ## Niko says that if the defocus is negative it will not perform CTF correction ## But it will also not correct the amplitudes particleparams = { "ptclnum": particle["particleNumber"], "psi": 0.0, "theta": 0.0, "phi": 0.0, "df1": -1.0, # workaround if no ctf correction "df2": -1.0, # set defocus to -1.0 Angstroms "angast": 0.0, "mag": 10000, # workaround to get around dstep "shx": 0.0, "shy": 0.0, "film": 1, "presa": 0.0, "dpres": 0.0, } imagedata = particle["particle"]["image"] if self.params["noctf"] is False: ctfdata, confidence = ctfdb.getBestCtfValueForImage( imagedata, msg=False, method=self.params["ctfmethod"] ) if ctfdata is not None: ### use defocus and astigmatism values particleparams["df1"] = abs(ctfdata["defocus1"] * 1e10) particleparams["df2"] = abs(ctfdata["defocus2"] * 1e10) particleparams["angast"] = ctfdata["angle_astigmatism"] # if using parameters from previous reconstruction if self.params["reconiterid"] is not None: emaneuler = self.getStackParticleEulersForIteration(particle["particleNumber"]) frealigneulers = apFrealign.convertEmanEulersToFrealign(emaneuler, sym=self.params["sym"]) particleparams["psi"] = frealigneulers["psi"] particleparams["theta"] = frealigneulers["theta"] particleparams["phi"] = frealigneulers["phi"] particleparams["shx"] = emaneuler["shiftx"] particleparams["shy"] = emaneuler["shifty"] if emaneuler["mirror"] is True: particleparams["shx"] *= -1 self.writeParticleParamLine(particleparams, f) f.close() return paramfile