def getBestCtfValue(self, imgdata, msg=False): if self.params['ctfrunid'] is not None: return ctfdb.getCtfValueForCtfRunId(imgdata, self.params['ctfrunid'], msg=msg) return ctfdb.getBestCtfValue(imgdata, sortType=self.params['ctfsorttype'], method=self.params['ctfmethod'], msg=msg)
def start(self): """ for each particle in the stack, get the information that RELION needs """ stackPartList = apStack.getStackParticlesFromId(self.params['stackid']) nptcls = len(stackPartList) currentImageId = stackPartList[0]['particle']['image'].dbid count = 0 imagenumber = 1 partParamsList = [] sys.stderr.write("reading stack particle data\n") #create list of dictionaries that will be passed to starFile maker later for stackPart in stackPartList: count += 1 if count % 100 == 0: sys.stderr.write(".") if count % 10000 == 0: sys.stderr.write("\nparticle %d of %d\n" % (count, nptcls)) # extra particle number information not read by Relion if count != stackPart['particleNumber']: apDisplay.printWarning( "particle number in database is not in sync") partParams = {} partParams['ptclnum'] = count ### get image data imagedata = stackPart['particle']['image'] if imagedata.dbid != currentImageId: imagenumber += 1 currentImageId = imagedata.dbid partParams['filmNum'] = imagenumber #print partParams['filmNum'] partParams['kv'] = imagedata['scope']['high tension'] / 1000.0 partParams['cs'] = imagedata['scope']['tem']['cs'] * 1000 ### get CTF data from image ctfdata = ctfdb.getBestCtfValue(imagedata, msg=False, sortType='maxconf') if ctfdata is not None: # use defocus & astigmatism values partParams['defocus1'] = abs(ctfdata['defocus1'] * 1e10) partParams['defocus2'] = abs(ctfdata['defocus2'] * 1e10) partParams['angle_astigmatism'] = ctfdata['angle_astigmatism'] partParams['amplitude_contrast'] = ctfdata[ 'amplitude_contrast'] else: apDisplay.printError( "No ctf information for particle %d in image %s" % (count, imagedata['filename'])) partParamsList.append(partParams) ###now make star file #first make header star = starFile.StarFile(self.params['outstar']) labels = [ '_rlnImageName', '_rlnMicrographName', '_rlnDefocusU', '_rlnDefocusV', '_rlnDefocusAngle', '_rlnVoltage', '_rlnSphericalAberration', '_rlnAmplitudeContrast', ] valueSets = [] #list of strings for star file ###now make particle data for partParams in partParamsList: relionDataLine = ( "%[email protected] mic%d %.6f %.6f %.6f %d %.6f %.6f" % ( partParams['ptclnum'], partParams['filmNum'], partParams['defocus2'], partParams['defocus1'], partParams['angle_astigmatism'], partParams['kv'], partParams['cs'], partParams['amplitude_contrast'], )) valueSets.append(relionDataLine) star.buildLoopFile("data_", labels, valueSets) star.write()
def getStackParticleParams(self): """ for each particle in the stack, get the information that RELION needs """ stackPartList = apStack.getStackParticlesFromId(self.params['stackid']) if 'last' not in self.params: self.params['last'] = len(stackPartList) firstImageId = stackPartList[0]['particle']['image'].dbid count = 0 lastImageId = -1 lastCtfData = None lastKv = -1 partParamsList = [] sys.stderr.write("reading stack particle data\n") t0 = time.time() for stackPart in stackPartList: count += 1 if count % 100 == 0: sys.stderr.write(".") if count % 10000 == 0: sys.stderr.write("\nparticle %d of %d\n"%(count, self.params['last'])) # extra particle number information not read by Relion if count != stackPart['particleNumber']: apDisplay.printWarning("particle number in database is not in sync") if count > self.params['last']: break partParams = {} partParams['ptclnum'] = count partParams['filmNum'] = self.getFilmNumber(stackPart, firstImageId) #print partParams['filmNum'] ### get image data imagedata = stackPart['particle']['image'] if self.originalStackData.defocpair is True: imagedata = apDefocalPairs.getDefocusPair(imagedata) if lastImageId == imagedata.dbid: ctfdata = lastCtfData partParams['kv'] = lastKv else: ctfdata = ctfdb.getBestCtfValue(imagedata, msg=False, method=self.params['ctfmethod']) partParams['kv'] = imagedata['scope']['high tension']/1000.0 lastCtfData = ctfdata lastImageId = imagedata.dbid lastKv = partParams['kv'] ### get CTF data from image if ctfdata is not None: # use defocus & astigmatism values partParams['defocus1'] = abs(ctfdata['defocus1']*1e10) partParams['defocus2'] = abs(ctfdata['defocus2']*1e10) partParams['angle_astigmatism'] = ctfdata['angle_astigmatism'] partParams['amplitude_contrast'] = ctfdata['amplitude_contrast'] else: apDisplay.printWarning("No ctf information for particle %d in image %d"%(count, imagedata.dbid)) partParams['defocus1'] = 0.1 partParams['defocus2'] = 0.1 partParams['angle_astigmatism'] = 0.0 partParams['amplitude_contrast'] = 0.07 if self.params['reconiterid'] is not None: eulerDict = self.getStackParticleEulersForIteration(stackPart) partParams.update(eulerDict) partParamsList.append(partParams) print "no class %d ; mismatch %d"%(self.noClassification, self.mismatch) sys.stderr.write("\ndone in %s\n\n"%(apDisplay.timeString(time.time()-t0))) return partParamsList
def start(self): """ for each particle in the stack, get the information that RELION needs """ stackPartList = apStack.getStackParticlesFromId(self.params['stackid']) nptcls=len(stackPartList) currentImageId = stackPartList[0]['particle']['image'].dbid count = 0 imagenumber=1 partParamsList = [] sys.stderr.write("reading stack particle data\n") #create list of dictionaries that will be passed to starFile maker later for stackPart in stackPartList: count += 1 if count % 100 == 0: sys.stderr.write(".") if count % 10000 == 0: sys.stderr.write("\nparticle %d of %d\n"%(count, nptcls)) # extra particle number information not read by Relion if count != stackPart['particleNumber']: apDisplay.printWarning("particle number in database is not in sync") partParams = {} partParams['ptclnum'] = count ### get image data imagedata = stackPart['particle']['image'] if imagedata.dbid != currentImageId: imagenumber+=1 currentImageId=imagedata.dbid partParams['filmNum'] = imagenumber #print partParams['filmNum'] partParams['kv'] = imagedata['scope']['high tension']/1000.0 partParams['cs'] =imagedata['scope']['tem']['cs']*1000 ### get CTF data from image ctfdata = ctfdb.getBestCtfValue(imagedata, msg=False, sortType='maxconf') if ctfdata is not None: # use defocus & astigmatism values partParams['defocus1'] = abs(ctfdata['defocus1']*1e10) partParams['defocus2'] = abs(ctfdata['defocus2']*1e10) partParams['angle_astigmatism'] = ctfdata['angle_astigmatism'] partParams['amplitude_contrast'] = ctfdata['amplitude_contrast'] else: apDisplay.printError("No ctf information for particle %d in image %s"%(count, imagedata['filename'])) partParamsList.append(partParams) ###now make star file #first make header star = starFile.StarFile(self.params['outstar']) labels = ['_rlnImageName', '_rlnMicrographName', '_rlnDefocusU', '_rlnDefocusV', '_rlnDefocusAngle', '_rlnVoltage', '_rlnSphericalAberration', '_rlnAmplitudeContrast', ] valueSets = [] #list of strings for star file ###now make particle data for partParams in partParamsList: relionDataLine = ("%[email protected] mic%d %.6f %.6f %.6f %d %.6f %.6f" %( partParams['ptclnum'], partParams['filmNum'], partParams['defocus2'], partParams['defocus1'], partParams['angle_astigmatism'], partParams['kv'], partParams['cs'], partParams['amplitude_contrast'], )) valueSets.append(relionDataLine) star.buildLoopFile( "data_", labels, valueSets ) star.write()
def getStackParticleParams(self): """ for each particle in the stack, get the information that RELION needs """ stackPartList = apStack.getStackParticlesFromId(self.params['stackid']) if 'last' not in self.params: self.params['last'] = len(stackPartList) firstImageId = stackPartList[0]['particle']['image'].dbid count = 0 lastImageId = -1 lastCtfData = None lastKv = -1 partParamsList = [] sys.stderr.write("reading stack particle data\n") t0 = time.time() for stackPart in stackPartList: count += 1 if count % 100 == 0: sys.stderr.write(".") if count % 10000 == 0: sys.stderr.write("\nparticle %d of %d\n" % (count, self.params['last'])) # extra particle number information not read by Relion if count != stackPart['particleNumber']: apDisplay.printWarning( "particle number in database is not in sync") if count > self.params['last']: break partParams = {} partParams['ptclnum'] = count partParams['filmNum'] = self.getFilmNumber(stackPart, firstImageId) #print partParams['filmNum'] ### get image data imagedata = stackPart['particle']['image'] if self.originalStackData.defocpair is True: imagedata = apDefocalPairs.getDefocusPair(imagedata) if lastImageId == imagedata.dbid: ctfdata = lastCtfData partParams['kv'] = lastKv else: ctfdata = ctfdb.getBestCtfValue( imagedata, msg=False, method=self.params['ctfmethod']) partParams['kv'] = imagedata['scope']['high tension'] / 1000.0 lastCtfData = ctfdata lastImageId = imagedata.dbid lastKv = partParams['kv'] ### get CTF data from image if ctfdata is not None: # use defocus & astigmatism values partParams['defocus1'] = abs(ctfdata['defocus1'] * 1e10) partParams['defocus2'] = abs(ctfdata['defocus2'] * 1e10) partParams['angle_astigmatism'] = ctfdata['angle_astigmatism'] partParams['amplitude_contrast'] = ctfdata[ 'amplitude_contrast'] else: apDisplay.printWarning( "No ctf information for particle %d in image %d" % (count, imagedata.dbid)) partParams['defocus1'] = 0.1 partParams['defocus2'] = 0.1 partParams['angle_astigmatism'] = 0.0 partParams['amplitude_contrast'] = 0.07 if self.params['reconiterid'] is not None: eulerDict = self.getStackParticleEulersForIteration(stackPart) partParams.update(eulerDict) partParamsList.append(partParams) print "no class %d ; mismatch %d" % (self.noClassification, self.mismatch) sys.stderr.write("\ndone in %s\n\n" % (apDisplay.timeString(time.time() - t0))) return partParamsList
def ctfCorrect(seriesname, rundir, projectid, sessionname, tiltseriesnumber, tiltfilename, frame_aligned_images, pixelsize, DefocusTol, iWidth, amp_contrast): """ Leginondb will be queried to get the 'best' defocus estimate on a per-image basis. Confident defoci will be gathered and unconfident defoci will be interpolated. Images will be CTF corrected by phase flipping using ctfphaseflip from the IMOD package. A plot of the defocus values will is made. A CTF plot using the mean defocus is made. """ try: apDisplay.printMsg('CTF correcting all tilt images using defocus values from Leginon database...') os.chdir(rundir) raw_path=rundir+'/raw/' ctfdir='%s/ctf_correction/' % rundir os.system("mkdir %s" % ctfdir) defocus_file_full=ctfdir+seriesname+'_defocus.txt' tilt_file_full=ctfdir+seriesname+'_tilts.txt' image_list_full=ctfdir+seriesname+'_images.txt' uncorrected_stack=ctfdir+'stack_uncorrected.mrc' corrected_stack=ctfdir+'stack_corrected.mrc' out_full=ctfdir+'out' log_file_full=ctfdir+'ctf_correction.log' project='ap'+projectid sinedon.setConfig('appiondata', db=project) sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata) tiltdata = apTomo.getImageList([tiltseriesdata]) frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata) tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images) cs = tiltdata[0]['scope']['tem']['cs']*1000 voltage = int(tiltdata[0]['scope']['high tension']/1000) if os.path.isfile(ctfdir+'out/out01.mrc'): #Throw exception if already ctf corrected sys.exit() #Get tilt azimuth cmd="awk '/TILT AZIMUTH/{print $3}' %s" % (tiltfilename) proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) (tilt_azimuth, err) = proc.communicate() tilt_azimuth=float(tilt_azimuth) estimated_defocus=[] for image in range(len(ordered_imagelist)): imgctfdata=ctfdb.getBestCtfValue(ordered_imagelist[image], msg=False) try: if imgctfdata['resolution_50_percent'] < 100.0: #if there's a yellow ring in Appion, trust defocus estimation estimated_defocus.append((imgctfdata['defocus1']+imgctfdata['defocus2'])*1000000000/2) else: #Poorly estimated. Guess its value later estimated_defocus.append(999999999) except: #No data. Guess its value later estimated_defocus.append(999999999) #Find mean and stdev to prune out confident defocus values that are way off defocus_stats_list=filter(lambda a: a != 999999999, estimated_defocus) avg=np.array(defocus_stats_list).mean() stdev=np.array(defocus_stats_list).std() good_tilts=[] good_defocus_list=[] for tilt, defocus in zip(tilts, estimated_defocus): if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev): good_defocus_list.append(defocus) good_tilts.append(tilt) #Using a linear best fit because quadratic and cubic go off the rails. Estimation doesn't need to be extremely accurate anyways. x=np.linspace(int(round(tilts[0])), int(round(tilts[len(tilts)-1])), 1000) s=scipy.interpolate.UnivariateSpline(good_tilts,good_defocus_list,k=1) y=s(x) #Make defocus list with good values and interpolations for bad values finished_defocus_list=[] for tilt, defocus in zip(tilts, estimated_defocus): if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev): finished_defocus_list.append(int(round(defocus))) else: #Interpolate finished_defocus_list.append(int(round(y[int(round(tilt))]))) new_avg=np.array(finished_defocus_list).mean() new_stdev=np.array(finished_defocus_list).std() #Write defocus file, tilt file, and image list file for ctfphaseflip and newstack f = open(defocus_file_full,'w') f.write("%d\t%d\t%.2f\t%.2f\t%d\t2\n" % (1,1,tilts[0],tilts[0],finished_defocus_list[0])) for i in range(1,len(tilts)): f.write("%d\t%d\t%.2f\t%.2f\t%d\n" % (i+1,i+1,tilts[i],tilts[i],finished_defocus_list[i])) f.close() f = open(tilt_file_full,'w') for tilt in tilts: f.write("%.2f\n" % tilt) f.close() mrc_list=[] presetname=tiltdata[0]['preset']['name'] for image in ordered_mrc_files: mrcname=presetname+image.split(presetname)[-1] mrc_list.append(raw_path+'/'+mrcname) f = open(image_list_full,'w') f.write("%d\n" % len(tilts)) for filename in mrc_list: f.write(filename+'\n') f.write("%d\n" % 0) f.close() #Rotate and pad images so that they are treated properly by ctfphaseflip. apDisplay.printMsg("Preparing images for IMOD...") for filename in mrc_list: image=mrc.read(filename) dimx=len(image[0]) dimy=len(image) #First rotate 90 degrees in counter-clockwise direction. This makes it so positive angle images are higher defocused on the right side of the image image=np.rot90(image, k=-1) #Rotate image and write image=imrotate(image, -tilt_azimuth, order=1) #Linear interpolation is fastest and there is barely a difference between linear and cubic mrc.write(image, filename) f = open(log_file_full,'w') #Make stack for correction,phase flip, extract images, replace images cmd1="newstack -fileinlist %s -output %s > %s" % (image_list_full, uncorrected_stack, log_file_full) f.write("%s\n\n" % cmd1) print cmd1 subprocess.check_call([cmd1], shell=True) cmd2="ctfphaseflip -input %s -output %s -AngleFile %s -defFn %s -pixelSize %s -volt %s -DefocusTol %s -iWidth %s -SphericalAberration %s -AmplitudeContrast %s 2>&1 | tee %s" % (uncorrected_stack, corrected_stack, tilt_file_full, defocus_file_full, pixelsize/10, voltage, DefocusTol, iWidth, cs, amp_contrast, log_file_full) f.write("\n\n%s\n\n" % cmd2) print cmd2 subprocess.check_call([cmd2], shell=True) cmd3="newstack -split 1 -append mrc %s %s >> %s" % (corrected_stack, out_full, log_file_full) f.write("\n\n%s\n\n" % cmd3) print cmd3 subprocess.check_call([cmd3], shell=True) f.write("\n\n") apDisplay.printMsg("Overwriting uncorrected raw images with CTF corrected images") new_images=glob.glob(ctfdir+'out*mrc') new_images.sort() #Unrotate and unpad images for filename in new_images: image=mrc.read(filename) image=imrotate(image, tilt_azimuth, order=1) image=np.rot90(image, k=1) big_dimx=len(image[0]) big_dimy=len(image) cropx1=int((big_dimx-dimx)/2) cropx2=int(dimx+(big_dimx-dimx)/2) cropy1=int((big_dimy-dimy)/2) cropy2=int(dimy+(big_dimy-dimy)/2) image=image[cropy1:cropy2,cropx1:cropx2] mrc.write(image, filename) for i in range(len(new_images)): cmd4="rm %s; ln %s %s" % (mrc_list[i], new_images[i], mrc_list[i]) f.write("%s\n" % cmd4) os.system(cmd4) #Make plots apProTomo2Aligner.makeDefocusPlot(rundir, seriesname, defocus_file_full) apProTomo2Aligner.makeCTFPlot(rundir, seriesname, defocus_file_full, voltage, cs) cleanup="rm %s %s" % (uncorrected_stack, corrected_stack) os.system(cleanup) output1="%.2f%% of the images for tilt-series #%s had poor defocus estimates or fell outside of one standard deviation from the original mean." % (100*(len(estimated_defocus)-len(defocus_stats_list))/len(estimated_defocus), tiltseriesnumber) output2="The defocus mean and standard deviation for tilt-series #%s after interpolating poor values is %.2f and %.2f microns, respectively." % (tiltseriesnumber, new_avg/1000, new_stdev/1000) f.write("\n");f.write(output1);f.write("\n");f.write(output2);f.write("\n");f.close() apDisplay.printMsg(output1) apDisplay.printMsg(output2) apDisplay.printMsg("CTF correction finished for tilt-series #%s!" % tiltseriesnumber) except subprocess.CalledProcessError: apDisplay.printError("An IMOD command failed, so CTF correction could not be completed. Make sure IMOD is in your $PATH.") except SystemExit: apDisplay.printWarning("It looks like you've already CTF corrected tilt-series #%s. Skipping CTF correction!" % tiltseriesnumber) except: apDisplay.printError("CTF correction could not be completed. Make sure IMOD, numpy, and scipy are in your $PATH. Make sure defocus has been estimated through Appion.\n")