def getBestCtfValue(self, imgdata, msg=False):
if self.params['ctfrunid'] is not None:
return ctfdb.getCtfValueForCtfRunId(imgdata,
self.params['ctfrunid'],
msg=msg)
return ctfdb.getBestCtfValue(imgdata,
sortType=self.params['ctfsorttype'],
method=self.params['ctfmethod'],
msg=msg)
def start(self):
"""
for each particle in the stack, get the information that RELION needs
"""
stackPartList = apStack.getStackParticlesFromId(self.params['stackid'])
nptcls = len(stackPartList)
currentImageId = stackPartList[0]['particle']['image'].dbid
count = 0
imagenumber = 1
partParamsList = []
sys.stderr.write("reading stack particle data\n")
#create list of dictionaries that will be passed to starFile maker later
for stackPart in stackPartList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
if count % 10000 == 0:
sys.stderr.write("\nparticle %d of %d\n" % (count, nptcls))
# extra particle number information not read by Relion
if count != stackPart['particleNumber']:
apDisplay.printWarning(
"particle number in database is not in sync")
partParams = {}
partParams['ptclnum'] = count
### get image data
imagedata = stackPart['particle']['image']
if imagedata.dbid != currentImageId:
imagenumber += 1
currentImageId = imagedata.dbid
partParams['filmNum'] = imagenumber
#print partParams['filmNum']
partParams['kv'] = imagedata['scope']['high tension'] / 1000.0
partParams['cs'] = imagedata['scope']['tem']['cs'] * 1000
### get CTF data from image
ctfdata = ctfdb.getBestCtfValue(imagedata,
msg=False,
sortType='maxconf')
if ctfdata is not None:
# use defocus & astigmatism values
partParams['defocus1'] = abs(ctfdata['defocus1'] * 1e10)
partParams['defocus2'] = abs(ctfdata['defocus2'] * 1e10)
partParams['angle_astigmatism'] = ctfdata['angle_astigmatism']
partParams['amplitude_contrast'] = ctfdata[
'amplitude_contrast']
else:
apDisplay.printError(
"No ctf information for particle %d in image %s" %
(count, imagedata['filename']))
partParamsList.append(partParams)
###now make star file
#first make header
star = starFile.StarFile(self.params['outstar'])
labels = [
'_rlnImageName',
'_rlnMicrographName',
'_rlnDefocusU',
'_rlnDefocusV',
'_rlnDefocusAngle',
'_rlnVoltage',
'_rlnSphericalAberration',
'_rlnAmplitudeContrast',
]
valueSets = [] #list of strings for star file
###now make particle data
for partParams in partParamsList:
relionDataLine = (
"%[email protected] mic%d %.6f %.6f %.6f %d %.6f %.6f" % (
partParams['ptclnum'],
partParams['filmNum'],
partParams['defocus2'],
partParams['defocus1'],
partParams['angle_astigmatism'],
partParams['kv'],
partParams['cs'],
partParams['amplitude_contrast'],
))
valueSets.append(relionDataLine)
star.buildLoopFile("data_", labels, valueSets)
star.write()
def getStackParticleParams(self):
"""
for each particle in the stack, get the information that RELION needs
"""
stackPartList = apStack.getStackParticlesFromId(self.params['stackid'])
if 'last' not in self.params:
self.params['last'] = len(stackPartList)
firstImageId = stackPartList[0]['particle']['image'].dbid
count = 0
lastImageId = -1
lastCtfData = None
lastKv = -1
partParamsList = []
sys.stderr.write("reading stack particle data\n")
t0 = time.time()
for stackPart in stackPartList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
if count % 10000 == 0:
sys.stderr.write("\nparticle %d of %d\n"%(count, self.params['last']))
# extra particle number information not read by Relion
if count != stackPart['particleNumber']:
apDisplay.printWarning("particle number in database is not in sync")
if count > self.params['last']:
break
partParams = {}
partParams['ptclnum'] = count
partParams['filmNum'] = self.getFilmNumber(stackPart, firstImageId)
#print partParams['filmNum']
### get image data
imagedata = stackPart['particle']['image']
if self.originalStackData.defocpair is True:
imagedata = apDefocalPairs.getDefocusPair(imagedata)
if lastImageId == imagedata.dbid:
ctfdata = lastCtfData
partParams['kv'] = lastKv
else:
ctfdata = ctfdb.getBestCtfValue(imagedata, msg=False, method=self.params['ctfmethod'])
partParams['kv'] = imagedata['scope']['high tension']/1000.0
lastCtfData = ctfdata
lastImageId = imagedata.dbid
lastKv = partParams['kv']
### get CTF data from image
if ctfdata is not None:
# use defocus & astigmatism values
partParams['defocus1'] = abs(ctfdata['defocus1']*1e10)
partParams['defocus2'] = abs(ctfdata['defocus2']*1e10)
partParams['angle_astigmatism'] = ctfdata['angle_astigmatism']
partParams['amplitude_contrast'] = ctfdata['amplitude_contrast']
else:
apDisplay.printWarning("No ctf information for particle %d in image %d"%(count, imagedata.dbid))
partParams['defocus1'] = 0.1
partParams['defocus2'] = 0.1
partParams['angle_astigmatism'] = 0.0
partParams['amplitude_contrast'] = 0.07
if self.params['reconiterid'] is not None:
eulerDict = self.getStackParticleEulersForIteration(stackPart)
partParams.update(eulerDict)
partParamsList.append(partParams)
print "no class %d ; mismatch %d"%(self.noClassification, self.mismatch)
sys.stderr.write("\ndone in %s\n\n"%(apDisplay.timeString(time.time()-t0)))
return partParamsList
def getBestCtfValue(self, imgdata, msg=False): if self.params['ctfrunid'] is not None: return ctfdb.getCtfValueForCtfRunId(imgdata, self.params['ctfrunid'], msg=msg) return ctfdb.getBestCtfValue(imgdata, sortType=self.params['ctfsorttype'], method=self.params['ctfmethod'], msg=msg)
def start(self):
"""
for each particle in the stack, get the information that RELION needs
"""
stackPartList = apStack.getStackParticlesFromId(self.params['stackid'])
nptcls=len(stackPartList)
currentImageId = stackPartList[0]['particle']['image'].dbid
count = 0
imagenumber=1
partParamsList = []
sys.stderr.write("reading stack particle data\n")
#create list of dictionaries that will be passed to starFile maker later
for stackPart in stackPartList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
if count % 10000 == 0:
sys.stderr.write("\nparticle %d of %d\n"%(count, nptcls))
# extra particle number information not read by Relion
if count != stackPart['particleNumber']:
apDisplay.printWarning("particle number in database is not in sync")
partParams = {}
partParams['ptclnum'] = count
### get image data
imagedata = stackPart['particle']['image']
if imagedata.dbid != currentImageId:
imagenumber+=1
currentImageId=imagedata.dbid
partParams['filmNum'] = imagenumber
#print partParams['filmNum']
partParams['kv'] = imagedata['scope']['high tension']/1000.0
partParams['cs'] =imagedata['scope']['tem']['cs']*1000
### get CTF data from image
ctfdata = ctfdb.getBestCtfValue(imagedata, msg=False, sortType='maxconf')
if ctfdata is not None:
# use defocus & astigmatism values
partParams['defocus1'] = abs(ctfdata['defocus1']*1e10)
partParams['defocus2'] = abs(ctfdata['defocus2']*1e10)
partParams['angle_astigmatism'] = ctfdata['angle_astigmatism']
partParams['amplitude_contrast'] = ctfdata['amplitude_contrast']
else:
apDisplay.printError("No ctf information for particle %d in image %s"%(count, imagedata['filename']))
partParamsList.append(partParams)
###now make star file
#first make header
star = starFile.StarFile(self.params['outstar'])
labels = ['_rlnImageName', '_rlnMicrographName',
'_rlnDefocusU', '_rlnDefocusV', '_rlnDefocusAngle', '_rlnVoltage',
'_rlnSphericalAberration', '_rlnAmplitudeContrast',
]
valueSets = [] #list of strings for star file
###now make particle data
for partParams in partParamsList:
relionDataLine = ("%[email protected] mic%d %.6f %.6f %.6f %d %.6f %.6f"
%( partParams['ptclnum'], partParams['filmNum'],
partParams['defocus2'], partParams['defocus1'], partParams['angle_astigmatism'],
partParams['kv'], partParams['cs'], partParams['amplitude_contrast'],
))
valueSets.append(relionDataLine)
star.buildLoopFile( "data_", labels, valueSets )
star.write()
def getStackParticleParams(self):
"""
for each particle in the stack, get the information that RELION needs
"""
stackPartList = apStack.getStackParticlesFromId(self.params['stackid'])
if 'last' not in self.params:
self.params['last'] = len(stackPartList)
firstImageId = stackPartList[0]['particle']['image'].dbid
count = 0
lastImageId = -1
lastCtfData = None
lastKv = -1
partParamsList = []
sys.stderr.write("reading stack particle data\n")
t0 = time.time()
for stackPart in stackPartList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
if count % 10000 == 0:
sys.stderr.write("\nparticle %d of %d\n" %
(count, self.params['last']))
# extra particle number information not read by Relion
if count != stackPart['particleNumber']:
apDisplay.printWarning(
"particle number in database is not in sync")
if count > self.params['last']:
break
partParams = {}
partParams['ptclnum'] = count
partParams['filmNum'] = self.getFilmNumber(stackPart, firstImageId)
#print partParams['filmNum']
### get image data
imagedata = stackPart['particle']['image']
if self.originalStackData.defocpair is True:
imagedata = apDefocalPairs.getDefocusPair(imagedata)
if lastImageId == imagedata.dbid:
ctfdata = lastCtfData
partParams['kv'] = lastKv
else:
ctfdata = ctfdb.getBestCtfValue(
imagedata, msg=False, method=self.params['ctfmethod'])
partParams['kv'] = imagedata['scope']['high tension'] / 1000.0
lastCtfData = ctfdata
lastImageId = imagedata.dbid
lastKv = partParams['kv']
### get CTF data from image
if ctfdata is not None:
# use defocus & astigmatism values
partParams['defocus1'] = abs(ctfdata['defocus1'] * 1e10)
partParams['defocus2'] = abs(ctfdata['defocus2'] * 1e10)
partParams['angle_astigmatism'] = ctfdata['angle_astigmatism']
partParams['amplitude_contrast'] = ctfdata[
'amplitude_contrast']
else:
apDisplay.printWarning(
"No ctf information for particle %d in image %d" %
(count, imagedata.dbid))
partParams['defocus1'] = 0.1
partParams['defocus2'] = 0.1
partParams['angle_astigmatism'] = 0.0
partParams['amplitude_contrast'] = 0.07
if self.params['reconiterid'] is not None:
eulerDict = self.getStackParticleEulersForIteration(stackPart)
partParams.update(eulerDict)
partParamsList.append(partParams)
print "no class %d ; mismatch %d" % (self.noClassification,
self.mismatch)
sys.stderr.write("\ndone in %s\n\n" %
(apDisplay.timeString(time.time() - t0)))
return partParamsList
def ctfCorrect(seriesname, rundir, projectid, sessionname, tiltseriesnumber, tiltfilename, frame_aligned_images, pixelsize, DefocusTol, iWidth, amp_contrast):
"""
Leginondb will be queried to get the 'best' defocus estimate on a per-image basis.
Confident defoci will be gathered and unconfident defoci will be interpolated.
Images will be CTF corrected by phase flipping using ctfphaseflip from the IMOD package.
A plot of the defocus values will is made.
A CTF plot using the mean defocus is made.
"""
try:
apDisplay.printMsg('CTF correcting all tilt images using defocus values from Leginon database...')
os.chdir(rundir)
raw_path=rundir+'/raw/'
ctfdir='%s/ctf_correction/' % rundir
os.system("mkdir %s" % ctfdir)
defocus_file_full=ctfdir+seriesname+'_defocus.txt'
tilt_file_full=ctfdir+seriesname+'_tilts.txt'
image_list_full=ctfdir+seriesname+'_images.txt'
uncorrected_stack=ctfdir+'stack_uncorrected.mrc'
corrected_stack=ctfdir+'stack_corrected.mrc'
out_full=ctfdir+'out'
log_file_full=ctfdir+'ctf_correction.log'
project='ap'+projectid
sinedon.setConfig('appiondata', db=project)
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata)
tiltdata = apTomo.getImageList([tiltseriesdata])
frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata)
tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images)
cs = tiltdata[0]['scope']['tem']['cs']*1000
voltage = int(tiltdata[0]['scope']['high tension']/1000)
if os.path.isfile(ctfdir+'out/out01.mrc'): #Throw exception if already ctf corrected
sys.exit()
#Get tilt azimuth
cmd="awk '/TILT AZIMUTH/{print $3}' %s" % (tiltfilename)
proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
(tilt_azimuth, err) = proc.communicate()
tilt_azimuth=float(tilt_azimuth)
estimated_defocus=[]
for image in range(len(ordered_imagelist)):
imgctfdata=ctfdb.getBestCtfValue(ordered_imagelist[image], msg=False)
try:
if imgctfdata['resolution_50_percent'] < 100.0: #if there's a yellow ring in Appion, trust defocus estimation
estimated_defocus.append((imgctfdata['defocus1']+imgctfdata['defocus2'])*1000000000/2)
else: #Poorly estimated. Guess its value later
estimated_defocus.append(999999999)
except: #No data. Guess its value later
estimated_defocus.append(999999999)
#Find mean and stdev to prune out confident defocus values that are way off
defocus_stats_list=filter(lambda a: a != 999999999, estimated_defocus)
avg=np.array(defocus_stats_list).mean()
stdev=np.array(defocus_stats_list).std()
good_tilts=[]
good_defocus_list=[]
for tilt, defocus in zip(tilts, estimated_defocus):
if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev):
good_defocus_list.append(defocus)
good_tilts.append(tilt)
#Using a linear best fit because quadratic and cubic go off the rails. Estimation doesn't need to be extremely accurate anyways.
x=np.linspace(int(round(tilts[0])), int(round(tilts[len(tilts)-1])), 1000)
s=scipy.interpolate.UnivariateSpline(good_tilts,good_defocus_list,k=1)
y=s(x)
#Make defocus list with good values and interpolations for bad values
finished_defocus_list=[]
for tilt, defocus in zip(tilts, estimated_defocus):
if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev):
finished_defocus_list.append(int(round(defocus)))
else: #Interpolate
finished_defocus_list.append(int(round(y[int(round(tilt))])))
new_avg=np.array(finished_defocus_list).mean()
new_stdev=np.array(finished_defocus_list).std()
#Write defocus file, tilt file, and image list file for ctfphaseflip and newstack
f = open(defocus_file_full,'w')
f.write("%d\t%d\t%.2f\t%.2f\t%d\t2\n" % (1,1,tilts[0],tilts[0],finished_defocus_list[0]))
for i in range(1,len(tilts)):
f.write("%d\t%d\t%.2f\t%.2f\t%d\n" % (i+1,i+1,tilts[i],tilts[i],finished_defocus_list[i]))
f.close()
f = open(tilt_file_full,'w')
for tilt in tilts:
f.write("%.2f\n" % tilt)
f.close()
mrc_list=[]
presetname=tiltdata[0]['preset']['name']
for image in ordered_mrc_files:
mrcname=presetname+image.split(presetname)[-1]
mrc_list.append(raw_path+'/'+mrcname)
f = open(image_list_full,'w')
f.write("%d\n" % len(tilts))
for filename in mrc_list:
f.write(filename+'\n')
f.write("%d\n" % 0)
f.close()
#Rotate and pad images so that they are treated properly by ctfphaseflip.
apDisplay.printMsg("Preparing images for IMOD...")
for filename in mrc_list:
image=mrc.read(filename)
dimx=len(image[0])
dimy=len(image)
#First rotate 90 degrees in counter-clockwise direction. This makes it so positive angle images are higher defocused on the right side of the image
image=np.rot90(image, k=-1)
#Rotate image and write
image=imrotate(image, -tilt_azimuth, order=1) #Linear interpolation is fastest and there is barely a difference between linear and cubic
mrc.write(image, filename)
f = open(log_file_full,'w')
#Make stack for correction,phase flip, extract images, replace images
cmd1="newstack -fileinlist %s -output %s > %s" % (image_list_full, uncorrected_stack, log_file_full)
f.write("%s\n\n" % cmd1)
print cmd1
subprocess.check_call([cmd1], shell=True)
cmd2="ctfphaseflip -input %s -output %s -AngleFile %s -defFn %s -pixelSize %s -volt %s -DefocusTol %s -iWidth %s -SphericalAberration %s -AmplitudeContrast %s 2>&1 | tee %s" % (uncorrected_stack, corrected_stack, tilt_file_full, defocus_file_full, pixelsize/10, voltage, DefocusTol, iWidth, cs, amp_contrast, log_file_full)
f.write("\n\n%s\n\n" % cmd2)
print cmd2
subprocess.check_call([cmd2], shell=True)
cmd3="newstack -split 1 -append mrc %s %s >> %s" % (corrected_stack, out_full, log_file_full)
f.write("\n\n%s\n\n" % cmd3)
print cmd3
subprocess.check_call([cmd3], shell=True)
f.write("\n\n")
apDisplay.printMsg("Overwriting uncorrected raw images with CTF corrected images")
new_images=glob.glob(ctfdir+'out*mrc')
new_images.sort()
#Unrotate and unpad images
for filename in new_images:
image=mrc.read(filename)
image=imrotate(image, tilt_azimuth, order=1)
image=np.rot90(image, k=1)
big_dimx=len(image[0])
big_dimy=len(image)
cropx1=int((big_dimx-dimx)/2)
cropx2=int(dimx+(big_dimx-dimx)/2)
cropy1=int((big_dimy-dimy)/2)
cropy2=int(dimy+(big_dimy-dimy)/2)
image=image[cropy1:cropy2,cropx1:cropx2]
mrc.write(image, filename)
for i in range(len(new_images)):
cmd4="rm %s; ln %s %s" % (mrc_list[i], new_images[i], mrc_list[i])
f.write("%s\n" % cmd4)
os.system(cmd4)
#Make plots
apProTomo2Aligner.makeDefocusPlot(rundir, seriesname, defocus_file_full)
apProTomo2Aligner.makeCTFPlot(rundir, seriesname, defocus_file_full, voltage, cs)
cleanup="rm %s %s" % (uncorrected_stack, corrected_stack)
os.system(cleanup)
output1="%.2f%% of the images for tilt-series #%s had poor defocus estimates or fell outside of one standard deviation from the original mean." % (100*(len(estimated_defocus)-len(defocus_stats_list))/len(estimated_defocus), tiltseriesnumber)
output2="The defocus mean and standard deviation for tilt-series #%s after interpolating poor values is %.2f and %.2f microns, respectively." % (tiltseriesnumber, new_avg/1000, new_stdev/1000)
f.write("\n");f.write(output1);f.write("\n");f.write(output2);f.write("\n");f.close()
apDisplay.printMsg(output1)
apDisplay.printMsg(output2)
apDisplay.printMsg("CTF correction finished for tilt-series #%s!" % tiltseriesnumber)
except subprocess.CalledProcessError:
apDisplay.printError("An IMOD command failed, so CTF correction could not be completed. Make sure IMOD is in your $PATH.")
except SystemExit:
apDisplay.printWarning("It looks like you've already CTF corrected tilt-series #%s. Skipping CTF correction!" % tiltseriesnumber)
except:
apDisplay.printError("CTF correction could not be completed. Make sure IMOD, numpy, and scipy are in your $PATH. Make sure defocus has been estimated through Appion.\n")
