def checkConflicts(self):
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.sessiondata = sessiondata
if self.params['goodalignerid'] is None:
if self.params['tiltseriesnumber'] is None :
apDisplay.printError("There is no tilt series specified")
if self.params['runname'] is None:
apDisplay.printError("enter a run name")
if self.params['description'] is None:
apDisplay.printError("enter a description, e.g. --description='awesome data'")
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseriesnumber'],sessiondata)
self.params['tiltseries'] = tiltdata
if self.params['othertilt'] is not None:
othertiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['othertilt'],sessiondata)
self.params['othertiltseries'] = othertiltdata
else:
self.params['othertiltseries'] = None
if self.params['alignmethod'] == 'leginon':
self.params['cycle'] = 0
self.params['region'] = 100
else:
self.createParamsFromGoodAligner(self.params['goodalignerid'])
if self.params['goodcycle'] is None:
if self.params['goodstart'] or self.params['goodend']:
apDisplay.printError("Cannot specify reseting image range without knowing the cycle number to reset to")
elif self.params['goodcycle'] >= self.params['cycle']:
apDisplay.printError("Only older cycle can be used as the base to reset image origin to")
# Sampling is needed for protomo1 and 2, not imod
if self.params['sample'] is not None:
if self.params['sample'] < 1:
apDisplay.printError("Sampling factor (--sample) must be greater or equal to 1.")
def checkConflicts(self):
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.sessiondata = sessiondata
if self.params['tiltseriesnumber'] is None :
apDisplay.printError("There is no tilt series specified")
if self.params['runname'] is None:
apDisplay.printError("enter a run name")
if self.params['description'] is None:
apDisplay.printError("enter a description, e.g. --description='awesome data'")
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseriesnumber'],sessiondata)
self.params['tiltseries'] = tiltdata
if self.params['othertilt'] is not None:
othertiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['othertilt'],sessiondata)
self.params['othertiltseries'] = othertiltdata
else:
self.params['othertiltseries'] = None
def checkConflicts(self):
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.sessiondata = sessiondata
if self.params['tiltseriesnumber'] is None :
apDisplay.printError("There is no tilt series specified")
if self.params['runname'] is None:
apDisplay.printError("enter a run name")
if self.params['description'] is None:
apDisplay.printError("enter a description, e.g. --description='awesome data'")
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseriesnumber'],sessiondata)
self.params['tiltseries'] = tiltdata
if self.params['othertilt'] is not None:
othertiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['othertilt'],sessiondata)
self.params['othertiltseries'] = othertiltdata
else:
self.params['othertiltseries'] = None
def getImageShapeFromTiltSeries(self):
sessiondata = apDatabase.getSessionDataFromSessionName(
self.params['sessionname'])
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(
self.params['tiltseriesnumber'], sessiondata)
firstimagedata = apTomo.getFirstImage(tiltdata)
self.imageshape = apTomo.getTomoImageShape(firstimagedata)
def start(self):
### some of this should go in preloop functions
###do queries
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata)
tiltseriessettings= apTomo.getTomographySettings(sessiondata,tiltseriesdata)
tiltdata=apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
print "ordering list"
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata)
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
###set up files
seriesname=self.params['seriesname']
print type(seriesname)
# param_out=seriesname+'.param'
###insert protomo run params
print "first insert"
protomodata = apProTomo.insertProtomoParams(seriesname)
print "second insert"
alignrun = apTomo.insertTomoAlignmentRun(sessiondata,tiltseriessettings,None,protomodata,None,1,self.params['runname'],self.params['rundir'],self.params['description'])
###insert protomo alignment
###hack to get around need to parse protomo param file
#should read imgref from tlt file
refineparamdict={'alismp':None,'alibox_x':None,'alibox_y':None,'cormod':None,'imgref':None}
###
self.params['goodcycle']=None
if self.params['goodrange'] is None:
self.params['goodstart']=1
self.params['goodend']=len(tilts)
alignerdata = apProTomo.insertAlignIteration(alignrun, protomodata, self.params, refineparamdict,ordered_imagelist[refimg])
# read tlt file
print "third insert"
alignmentdict, geometrydict, seriesname = apProTomo.parseTilt(self.params['tltfile'])
# insert geometry model
modeldata = apProTomo.insertModel2(alignerdata, geometrydict)
#insert image alignments
for i,imagedata in enumerate(ordered_imagelist):
#Caution...assumes ordered_imagelist is in same order as tlt file
apProTomo.insertTiltAlignment(alignerdata,imagedata,i,alignmentdict[i+1],center=None)
print "fourth insert"
apTomo.insertTiltsInAlignRun(alignrun, tiltseriesdata,tiltseriessettings,True)
def setRunDir(self):
sessiondata = apDatabase.getSessionDataFromSessionName(self.params["sessionname"])
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params["tiltseriesnumber"], sessiondata)
path = self.getDefaultBaseAppionDir(sessiondata, ["tomo"])
tiltseriespath = "tiltseries%d" % self.params["tiltseriesnumber"]
if self.params["full"]:
tomovolumepath = ""
else:
tomovolumepath = self.params["volume"]
intermediatepath = os.path.join(tiltseriespath, self.params["runname"], tomovolumepath)
self.params["rundir"] = os.path.join(path, intermediatepath)
self.params["aligndir"] = os.path.abspath(os.path.join(path, tiltseriespath, "align", self.params["runname"]))
def setRunDir(self):
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseriesnumber'],sessiondata)
path = self.getDefaultBaseAppionDir(sessiondata,['tomo'])
tiltseriespath = "tiltseries%d" % self.params['tiltseriesnumber']
if self.params['full']:
tomovolumepath = ""
else:
tomovolumepath = self.params['volume']
intermediatepath = os.path.join(tiltseriespath,self.params['runname'],tomovolumepath)
self.params['rundir'] = os.path.join(path,intermediatepath)
self.params['aligndir'] = os.path.abspath(os.path.join(path,tiltseriespath,'align',self.params['runname']))
def setRunDir(self): sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseriesnumber'],sessiondata) path = self.getDefaultBaseAppionDir(sessiondata,['tomo']) tiltseriespath = "tiltseries%d" % self.params['tiltseriesnumber'] if self.params['full']: tomovolumepath = "" else: tomovolumepath = self.params['volume'] intermediatepath = os.path.join(tiltseriespath,self.params['runname'],tomovolumepath) self.params['rundir'] = os.path.join(path,intermediatepath) self.params['aligndir'] = os.path.abspath(os.path.join(path,tiltseriespath,'align',self.params['runname']))
def start(self):
### some of this should go in preloop functions
###do queries
os.chdir(self.params['rundir'])
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata)
tiltdata = apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata)
def uploadTomo(params):
if not params['commit']:
apDisplay.printWarning("not commiting tomogram to database")
return
apDisplay.printMsg("Commiting tomogram to database")
sessiondata = apDatabase.getSessionDataFromSessionName(params['sessionname'])
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(params['tiltseriesnumber'],sessiondata)
runname = params['runname']
name = params['name']
if params['full']:
fullbin = params['bin']
else:
fullbin = 1
subbin = params['bin']
alignrun = insertTomoAlignmentRun(sessiondata,None,None,None,None,fullbin,runname,params['aligndir'],'manual alignment from upload')
# only tilt series in one alignrun for now
insertTiltsInAlignRun(alignrun, tiltdata,None,True)
alignerdata = insertAlignerParams(alignrun,params)
firstimagedata = getFirstImage(tiltdata)
path = os.path.abspath(params['rundir'])
description = params['description']
if params['full']:
thickness = params['shape'][0] * fullbin
uploadfile = params['zprojfile']
projectimagedata = uploadZProjection(runname,firstimagedata,uploadfile)
fullrundata = insertFullTomoRun(sessiondata,path,runname,'upload')
return insertFullTomogram(sessiondata,tiltdata,alignerdata,fullrundata,name,description,projectimagedata,thickness,None,fullbin,[])
else:
projectimagedata = None
fulltomopath = params['rundir'].replace('/'+params['volume'],'')
dummyname = 'dummy'
dummydescription = 'fake full tomogram for subtomogram upload'
thickness = params['shape'][0] * subbin
fullrundata = insertFullTomoRun(sessiondata,fulltomopath,runname,'upload')
fulltomogram = insertFullTomogram(sessiondata,tiltdata,alignerdata,fullrundata,dummyname,dummydescription,projectimagedata,thickness,None,fullbin,[])
apix = apDatabase.getPixelSize(firstimagedata)
tomoq = appiondata.ApTomogramData()
tomoq['session'] = sessiondata
tomoq['tiltseries'] = tiltdata
results = tomoq.query()
index = len(results)+1
pixelsize = 1e-10 * apix * subbin
runname = params['volume']
shape = map((lambda x: x * subbin), params['shape'])
dimension = {'x':shape[2],'y':shape[1], 'z':shape[0]}
subtomorundata = insertSubTomoRun(sessiondata,
None,None,runname,params['invert'],subbin)
return insertSubTomogram(fulltomogram,subtomorundata,None,0,dimension,path,name,index,pixelsize,description)
def start(self):
###do queries
sessiondata = apDatabase.getSessionDataFromSessionName(
self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(
self.params['tiltseries'], sessiondata)
tiltseriessettings = apTomo.getTomographySettings(
sessiondata, tiltseriesdata)
tiltdata = apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
print "ordering list"
tilts, ordered_imagelist, ordered_mrc_files, refimg = apTomo.orderImageList(
tiltdata)
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
###set up files
seriesname = 'series' + str(self.params['tiltseries'])
tiltfilename = seriesname + '.tlt'
param_out = seriesname + '.param'
maxtilt = max([abs(tilts[0]), abs(tilts[-1])])
apDisplay.printMsg("highest tilt angle is %f" % maxtilt)
self.params['cos_alpha'] = math.cos(maxtilt * math.pi / 180)
self.params['raw_path'] = os.path.join(self.params['rundir'], 'raw')
rawexists = apParam.createDirectory(self.params['raw_path'])
apDisplay.printMsg("copying raw images")
newfilenames = apProTomo.getImageFiles(ordered_imagelist,
self.params['raw_path'],
link=False)
#get alignment data
alignerdata = apTomo.getAlignerdata(self.params['alignerid'])
imgshape = apTomo.getTomoImageShape(ordered_imagelist[0])
imgcenter = {'x': self.imgshape[1] / 2, 'y': self.imgshape[0] / 2}
specimen_euler, azimuth, origins, rotations = apTomo.getAlignmentFromDB(
alignerdata, imgcenter)
#write protomo2 tilt file
outtltfile = 'series.tlt'
seriesname = 'series'
apProTomo.writeTiltFile2(outfilename, seriesname, specimen_eulers,
azimuth, referenceimage)
def start(self):
### some of this should go in preloop functions
###do queries
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata)
tiltdata=apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata)
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
###set up files
seriesname='series'+str(self.params['tiltseries'])
tiltfilename=seriesname+'.tlt'
param_out=seriesname+'.param'
maxtilt=max([abs(tilts[0]),abs(tilts[-1])])
apDisplay.printMsg("highest tilt angle is %f" % maxtilt)
self.params['cos_alpha']=math.cos(maxtilt*math.pi/180)
self.params['raw_path']=os.path.join(self.params['rundir'],'raw')
rawexists=apParam.createDirectory(self.params['raw_path'])
apDisplay.printMsg("copying raw images")
newfilenames=apProTomo.getImageFiles(ordered_imagelist,self.params['raw_path'], link=False)
###create tilt file
#get image size from the first image
imagesizex=tiltdata[0]['image'].shape[0]
imagesizey=tiltdata[0]['image'].shape[1]
#shift half tilt series relative to eachother
#SS I'm arbitrarily making the bin parameter here 1 because it's not necessary to sample at this point
shifts = apTomo.getGlobalShift(ordered_imagelist, 1, refimg)
#OPTION: refinement might be more robust by doing one round of IMOD aligment to prealign images before doing protomo refine
origins=apProTomo2Prep.convertShiftsToOrigin(shifts, imagesizex, imagesizey)
#determine azimuth
azimuth=apTomo.getAverageAzimuthFromSeries(ordered_imagelist)
apProTomo2Prep.writeTileFile2(tiltfilename, seriesname, newfilenames, origins, tilts, azimuth, refimg)
def prepareTiltFile(sessionname, seriesname, tiltfilename, tiltseriesnumber, raw_path, frame_aligned_images, link=False, coarse=True):
'''
Creates tlt file from basic image information and copies raw images
'''
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata)
tiltdata = apTomo.getImageList([tiltseriesdata])
apDisplay.printMsg("getting imagelist")
frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata)
tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images)
if frame_aligned_images == "True": #Azimuth is only present in the non-frame aligned images
a,ordered_imagelist_for_azimuth,c,d,e = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned="False")
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
maxtilt = max([abs(tilts[0]),abs(tilts[-1])])
apDisplay.printMsg("highest tilt angle is %f" % maxtilt)
if coarse == "True":
if frame_aligned_images == "True": #Azimuth is only present in the non-frame aligned images
azimuth = apTomo.getAverageAzimuthFromSeries(ordered_imagelist_for_azimuth)
else:
azimuth = apTomo.getAverageAzimuthFromSeries(ordered_imagelist)
rawexists = apParam.createDirectory(raw_path)
apDisplay.printMsg("Copying raw images, y-flipping, normalizing, and converting images to float32 for Protomo...") #Linking removed because raw images need to be y-flipped for Protomo:(.
newfilenames, new_ordered_imagelist = apProTomo.getImageFiles(ordered_imagelist, raw_path, link=False, copy="True")
###create tilt file
#get image size from the first image
imagesizex = tiltdata[0]['image'].shape[1]
imagesizey = tiltdata[0]['image'].shape[0]
#shift half tilt series relative to eachother
#SS I'm arbitrarily making the bin parameter here 1 because it's not necessary to sample at this point
shifts = apTomo.getGlobalShift(ordered_imagelist, 1, refimg)
#OPTION: refinement might be more robust by doing one round of IMOD aligment to prealign images before doing protomo refine
origins = convertShiftsToOrigin(shifts, imagesizex, imagesizey)
writeTiltFile2(tiltfilename, seriesname, newfilenames, origins, tilts, azimuth, refimg)
return tilts, accumulated_dose_list, new_ordered_imagelist, maxtilt
def doseCompensate(seriesname, rundir, sessionname, tiltseriesnumber, frame_aligned_images, raw_path, pixelsize, dose_presets, dose_a, dose_b, dose_c):
"""
Images will be lowpass filtered using equation (3) from Grant & Grigorieff, 2015.
No changes to the database are made. No backups are made.
"""
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata)
tiltdata = apTomo.getImageList([tiltseriesdata])
frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata)
tilts, ordered_imagelist, accumulated_dose_list, ordered_mrc_files, refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned="False")
if frame_aligned_images == "True": #For different image filenames
a, ordered_imagelist, c, d, e = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images)
newfilenames, new_ordered_imagelist = apProTomo.getImageFiles(ordered_imagelist, raw_path, link=False, copy=False)
if (dose_presets == "Light"):
dose_a = 0.245
dose_b = -1.6
dose_c = 12
elif (dose_presets == "Moderate"):
dose_a = 0.245
dose_b = -1.665
dose_c = 2.81
elif (dose_presets == "Heavy"):
dose_a = 0.245
dose_b = -1.4
dose_c = 2
apDisplay.printMsg('Dose compensating all tilt images with a=%s, b=%s, and c=%s...' % (dose_a, dose_b, dose_c))
for image, j in zip(new_ordered_imagelist, range(len(new_ordered_imagelist))):
lowpass = float(np.real(complex(dose_a/(accumulated_dose_list[j] - dose_c))**(1/dose_b))) #equation (3) from Grant & Grigorieff, 2015
if lowpass < 0.0:
lowpass = 0.0
im = mrc.read(image)
im = imagefilter.lowPassFilter(im, apix=pixelsize, radius=lowpass, msg=False)
im=imagenorm.normStdev(im)
mrc.write(im, image)
#Make plots
apProTomo2Aligner.makeDosePlots(rundir, seriesname, tilts, accumulated_dose_list, dose_a, dose_b, dose_c)
apDisplay.printMsg("Dose compensation finished for tilt-series #%s!" % tiltseriesnumber)
return
def start(self):
###do queries
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata)
tiltseriessettings= apTomo.getTomographySettings(sessiondata,tiltseriesdata)
tiltdata=apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
print "ordering list"
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata)
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
###set up files
seriesname='series'+str(self.params['tiltseries'])
tiltfilename=seriesname+'.tlt'
param_out=seriesname+'.param'
maxtilt=max([abs(tilts[0]),abs(tilts[-1])])
apDisplay.printMsg("highest tilt angle is %f" % maxtilt)
self.params['cos_alpha']=math.cos(maxtilt*math.pi/180)
self.params['raw_path']=os.path.join(self.params['rundir'],'raw')
rawexists=apParam.createDirectory(self.params['raw_path'])
apDisplay.printMsg("copying raw images")
newfilenames=apProTomo.getImageFiles(ordered_imagelist,self.params['raw_path'], link=False)
#get alignment data
alignerdata = apTomo.getAlignerdata(self.params['alignerid'])
imgshape = apTomo.getTomoImageShape(ordered_imagelist[0])
imgcenter = {'x':self.imgshape[1]/2,'y':self.imgshape[0]/2}
specimen_euler, azimuth, origins, rotations = apTomo.getAlignmentFromDB(alignerdata,imgcenter)
#write protomo2 tilt file
outtltfile='series.tlt'
seriesname='series'
apProTomo.writeTiltFile2(outfilename, seriesname, specimen_eulers, azimuth, referenceimage )
def getImageShapeFromTiltSeries(self):
sessiondata = apDatabase.getSessionDataFromSessionName(self.params["sessionname"])
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params["tiltseriesnumber"], sessiondata)
firstimagedata = apTomo.getFirstImage(tiltdata)
self.imageshape = apTomo.getTomoImageShape(firstimagedata)
def ctfCorrect(seriesname, rundir, projectid, sessionname, tiltseriesnumber, tiltfilename, frame_aligned_images, pixelsize, DefocusTol, iWidth, amp_contrast):
"""
Leginondb will be queried to get the 'best' defocus estimate on a per-image basis.
Confident defoci will be gathered and unconfident defoci will be interpolated.
Images will be CTF corrected by phase flipping using ctfphaseflip from the IMOD package.
A plot of the defocus values will is made.
A CTF plot using the mean defocus is made.
"""
try:
apDisplay.printMsg('CTF correcting all tilt images using defocus values from Leginon database...')
os.chdir(rundir)
raw_path=rundir+'/raw/'
ctfdir='%s/ctf_correction/' % rundir
os.system("mkdir %s" % ctfdir)
defocus_file_full=ctfdir+seriesname+'_defocus.txt'
tilt_file_full=ctfdir+seriesname+'_tilts.txt'
image_list_full=ctfdir+seriesname+'_images.txt'
uncorrected_stack=ctfdir+'stack_uncorrected.mrc'
corrected_stack=ctfdir+'stack_corrected.mrc'
out_full=ctfdir+'out'
log_file_full=ctfdir+'ctf_correction.log'
project='ap'+projectid
sinedon.setConfig('appiondata', db=project)
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata)
tiltdata = apTomo.getImageList([tiltseriesdata])
frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata)
tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images)
cs = tiltdata[0]['scope']['tem']['cs']*1000
voltage = int(tiltdata[0]['scope']['high tension']/1000)
if os.path.isfile(ctfdir+'out/out01.mrc'): #Throw exception if already ctf corrected
sys.exit()
#Get tilt azimuth
cmd="awk '/TILT AZIMUTH/{print $3}' %s" % (tiltfilename)
proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
(tilt_azimuth, err) = proc.communicate()
tilt_azimuth=float(tilt_azimuth)
estimated_defocus=[]
for image in range(len(ordered_imagelist)):
imgctfdata=ctfdb.getBestCtfValue(ordered_imagelist[image], msg=False)
try:
if imgctfdata['resolution_50_percent'] < 100.0: #if there's a yellow ring in Appion, trust defocus estimation
estimated_defocus.append((imgctfdata['defocus1']+imgctfdata['defocus2'])*1000000000/2)
else: #Poorly estimated. Guess its value later
estimated_defocus.append(999999999)
except: #No data. Guess its value later
estimated_defocus.append(999999999)
#Find mean and stdev to prune out confident defocus values that are way off
defocus_stats_list=filter(lambda a: a != 999999999, estimated_defocus)
avg=np.array(defocus_stats_list).mean()
stdev=np.array(defocus_stats_list).std()
good_tilts=[]
good_defocus_list=[]
for tilt, defocus in zip(tilts, estimated_defocus):
if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev):
good_defocus_list.append(defocus)
good_tilts.append(tilt)
#Using a linear best fit because quadratic and cubic go off the rails. Estimation doesn't need to be extremely accurate anyways.
x=np.linspace(int(round(tilts[0])), int(round(tilts[len(tilts)-1])), 1000)
s=scipy.interpolate.UnivariateSpline(good_tilts,good_defocus_list,k=1)
y=s(x)
#Make defocus list with good values and interpolations for bad values
finished_defocus_list=[]
for tilt, defocus in zip(tilts, estimated_defocus):
if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev):
finished_defocus_list.append(int(round(defocus)))
else: #Interpolate
finished_defocus_list.append(int(round(y[int(round(tilt))])))
new_avg=np.array(finished_defocus_list).mean()
new_stdev=np.array(finished_defocus_list).std()
#Write defocus file, tilt file, and image list file for ctfphaseflip and newstack
f = open(defocus_file_full,'w')
f.write("%d\t%d\t%.2f\t%.2f\t%d\t2\n" % (1,1,tilts[0],tilts[0],finished_defocus_list[0]))
for i in range(1,len(tilts)):
f.write("%d\t%d\t%.2f\t%.2f\t%d\n" % (i+1,i+1,tilts[i],tilts[i],finished_defocus_list[i]))
f.close()
f = open(tilt_file_full,'w')
for tilt in tilts:
f.write("%.2f\n" % tilt)
f.close()
mrc_list=[]
presetname=tiltdata[0]['preset']['name']
for image in ordered_mrc_files:
mrcname=presetname+image.split(presetname)[-1]
mrc_list.append(raw_path+'/'+mrcname)
f = open(image_list_full,'w')
f.write("%d\n" % len(tilts))
for filename in mrc_list:
f.write(filename+'\n')
f.write("%d\n" % 0)
f.close()
#Rotate and pad images so that they are treated properly by ctfphaseflip.
apDisplay.printMsg("Preparing images for IMOD...")
for filename in mrc_list:
image=mrc.read(filename)
dimx=len(image[0])
dimy=len(image)
#First rotate 90 degrees in counter-clockwise direction. This makes it so positive angle images are higher defocused on the right side of the image
image=np.rot90(image, k=-1)
#Rotate image and write
image=imrotate(image, -tilt_azimuth, order=1) #Linear interpolation is fastest and there is barely a difference between linear and cubic
mrc.write(image, filename)
f = open(log_file_full,'w')
#Make stack for correction,phase flip, extract images, replace images
cmd1="newstack -fileinlist %s -output %s > %s" % (image_list_full, uncorrected_stack, log_file_full)
f.write("%s\n\n" % cmd1)
print cmd1
subprocess.check_call([cmd1], shell=True)
cmd2="ctfphaseflip -input %s -output %s -AngleFile %s -defFn %s -pixelSize %s -volt %s -DefocusTol %s -iWidth %s -SphericalAberration %s -AmplitudeContrast %s 2>&1 | tee %s" % (uncorrected_stack, corrected_stack, tilt_file_full, defocus_file_full, pixelsize/10, voltage, DefocusTol, iWidth, cs, amp_contrast, log_file_full)
f.write("\n\n%s\n\n" % cmd2)
print cmd2
subprocess.check_call([cmd2], shell=True)
cmd3="newstack -split 1 -append mrc %s %s >> %s" % (corrected_stack, out_full, log_file_full)
f.write("\n\n%s\n\n" % cmd3)
print cmd3
subprocess.check_call([cmd3], shell=True)
f.write("\n\n")
apDisplay.printMsg("Overwriting uncorrected raw images with CTF corrected images")
new_images=glob.glob(ctfdir+'out*mrc')
new_images.sort()
#Unrotate and unpad images
for filename in new_images:
image=mrc.read(filename)
image=imrotate(image, tilt_azimuth, order=1)
image=np.rot90(image, k=1)
big_dimx=len(image[0])
big_dimy=len(image)
cropx1=int((big_dimx-dimx)/2)
cropx2=int(dimx+(big_dimx-dimx)/2)
cropy1=int((big_dimy-dimy)/2)
cropy2=int(dimy+(big_dimy-dimy)/2)
image=image[cropy1:cropy2,cropx1:cropx2]
mrc.write(image, filename)
for i in range(len(new_images)):
cmd4="rm %s; ln %s %s" % (mrc_list[i], new_images[i], mrc_list[i])
f.write("%s\n" % cmd4)
os.system(cmd4)
#Make plots
apProTomo2Aligner.makeDefocusPlot(rundir, seriesname, defocus_file_full)
apProTomo2Aligner.makeCTFPlot(rundir, seriesname, defocus_file_full, voltage, cs)
cleanup="rm %s %s" % (uncorrected_stack, corrected_stack)
os.system(cleanup)
output1="%.2f%% of the images for tilt-series #%s had poor defocus estimates or fell outside of one standard deviation from the original mean." % (100*(len(estimated_defocus)-len(defocus_stats_list))/len(estimated_defocus), tiltseriesnumber)
output2="The defocus mean and standard deviation for tilt-series #%s after interpolating poor values is %.2f and %.2f microns, respectively." % (tiltseriesnumber, new_avg/1000, new_stdev/1000)
f.write("\n");f.write(output1);f.write("\n");f.write(output2);f.write("\n");f.close()
apDisplay.printMsg(output1)
apDisplay.printMsg(output2)
apDisplay.printMsg("CTF correction finished for tilt-series #%s!" % tiltseriesnumber)
except subprocess.CalledProcessError:
apDisplay.printError("An IMOD command failed, so CTF correction could not be completed. Make sure IMOD is in your $PATH.")
except SystemExit:
apDisplay.printWarning("It looks like you've already CTF corrected tilt-series #%s. Skipping CTF correction!" % tiltseriesnumber)
except:
apDisplay.printError("CTF correction could not be completed. Make sure IMOD, numpy, and scipy are in your $PATH. Make sure defocus has been estimated through Appion.\n")
def start(self):
### some of this should go in preloop functions
###do queries
sessiondata = apDatabase.getSessionDataFromSessionName(
self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(
self.params['tiltseries'], sessiondata)
tiltseriessettings = apTomo.getTomographySettings(
sessiondata, tiltseriesdata)
tiltdata = apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
print "ordering list"
tilts, ordered_imagelist, ordered_mrc_files, refimg = apTomo.orderImageList(
tiltdata)
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
###set up files
seriesname = self.params['seriesname']
print type(seriesname)
# param_out=seriesname+'.param'
###insert protomo run params
print "first insert"
protomodata = apProTomo.insertProtomoParams(seriesname)
print "second insert"
alignrun = apTomo.insertTomoAlignmentRun(sessiondata,
tiltseriessettings, None,
protomodata, None, 1,
self.params['runname'],
self.params['rundir'],
self.params['description'])
###insert protomo alignment
###hack to get around need to parse protomo param file
#should read imgref from tlt file
refineparamdict = {
'alismp': None,
'alibox_x': None,
'alibox_y': None,
'cormod': None,
'imgref': None
}
###
self.params['goodcycle'] = None
if self.params['goodrange'] is None:
self.params['goodstart'] = 1
self.params['goodend'] = len(tilts)
alignerdata = apProTomo.insertAlignIteration(alignrun, protomodata,
self.params,
refineparamdict,
ordered_imagelist[refimg])
# read tlt file
print "third insert"
alignmentdict, geometrydict, seriesname = apProTomo.parseTilt(
self.params['tltfile'])
# insert geometry model
modeldata = apProTomo.insertModel2(alignerdata, geometrydict)
#insert image alignments
for i, imagedata in enumerate(ordered_imagelist):
#Caution...assumes ordered_imagelist is in same order as tlt file
apProTomo.insertTiltAlignment(alignerdata,
imagedata,
i,
alignmentdict[i + 1],
center=None)
print "fourth insert"
apTomo.insertTiltsInAlignRun(alignrun, tiltseriesdata,
tiltseriessettings, True)
def uploadTomo(params):
if not params["commit"]:
apDisplay.printWarning("not commiting tomogram to database")
return
apDisplay.printMsg("Commiting tomogram to database")
sessiondata = apDatabase.getSessionDataFromSessionName(params["sessionname"])
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(params["tiltseriesnumber"], sessiondata)
runname = params["runname"]
name = params["name"]
if params["full"]:
fullbin = params["bin"]
else:
fullbin = 1
subbin = params["bin"]
alignrun = insertTomoAlignmentRun(
sessiondata, None, None, None, None, fullbin, runname, params["aligndir"], "manual alignment from upload"
)
# only tilt series in one alignrun for now
insertTiltsInAlignRun(alignrun, tiltdata, None, True)
alignerdata = insertAlignerParams(alignrun, params)
firstimagedata = getFirstImage(tiltdata)
path = os.path.abspath(params["rundir"])
description = params["description"]
if params["full"]:
thickness = params["shape"][0] * fullbin
uploadfile = params["zprojfile"]
projectimagedata = uploadZProjection(runname, firstimagedata, uploadfile)
fullrundata = insertFullTomoRun(sessiondata, path, runname, "upload")
return insertFullTomogram(
sessiondata,
tiltdata,
alignerdata,
fullrundata,
name,
description,
projectimagedata,
thickness,
None,
fullbin,
[],
)
else:
projectimagedata = None
fulltomopath = params["rundir"].replace("/" + params["volume"], "")
dummyname = "dummy"
dummydescription = "fake full tomogram for subtomogram upload"
thickness = params["shape"][0] * subbin
fullrundata = insertFullTomoRun(sessiondata, fulltomopath, runname, "upload")
fulltomogram = insertFullTomogram(
sessiondata,
tiltdata,
alignerdata,
fullrundata,
dummyname,
dummydescription,
projectimagedata,
thickness,
None,
fullbin,
[],
)
apix = apDatabase.getPixelSize(firstimagedata)
tomoq = appiondata.ApTomogramData()
tomoq["session"] = sessiondata
tomoq["tiltseries"] = tiltdata
results = tomoq.query()
index = len(results) + 1
pixelsize = 1e-10 * apix * subbin
runname = params["volume"]
shape = map((lambda x: x * subbin), params["shape"])
dimension = {"x": shape[2], "y": shape[1], "z": shape[0]}
subtomorundata = insertSubTomoRun(sessiondata, None, None, runname, params["invert"], subbin)
return insertSubTomogram(
fulltomogram, subtomorundata, None, 0, dimension, path, name, index, pixelsize, description
)
def uploadTomo(params):
if not params['commit']:
apDisplay.printWarning("not commiting tomogram to database")
return
apDisplay.printMsg("Commiting tomogram to database")
sessiondata = apDatabase.getSessionDataFromSessionName(
params['sessionname'])
tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(
params['tiltseriesnumber'], sessiondata)
runname = params['runname']
name = params['name']
if params['full']:
fullbin = params['bin']
else:
fullbin = 1
subbin = params['bin']
alignrun = insertTomoAlignmentRun(sessiondata, None, None, None, None,
fullbin, runname, params['aligndir'],
'manual alignment from upload')
# only tilt series in one alignrun for now
insertTiltsInAlignRun(alignrun, tiltdata, None, True)
alignerdata = insertAlignerParams(alignrun, params)
firstimagedata = getFirstImage(tiltdata)
path = os.path.abspath(params['rundir'])
description = params['description']
if params['full']:
thickness = params['shape'][0] * fullbin
uploadfile = params['zprojfile']
projectimagedata = uploadZProjection(runname, firstimagedata,
uploadfile)
fullrundata = insertFullTomoRun(sessiondata, path, runname, 'upload')
return insertFullTomogram(sessiondata, tiltdata, alignerdata,
fullrundata, name, description,
projectimagedata, thickness, None, fullbin,
[])
else:
projectimagedata = None
fulltomopath = params['rundir'].replace('/' + params['volume'], '')
dummyname = 'dummy'
dummydescription = 'fake full tomogram for subtomogram upload'
thickness = params['shape'][0] * subbin
fullrundata = insertFullTomoRun(sessiondata, fulltomopath, runname,
'upload')
fulltomogram = insertFullTomogram(sessiondata, tiltdata, alignerdata,
fullrundata, dummyname,
dummydescription, projectimagedata,
thickness, None, fullbin, [])
apix = apDatabase.getPixelSize(firstimagedata)
tomoq = appiondata.ApTomogramData()
tomoq['session'] = sessiondata
tomoq['tiltseries'] = tiltdata
results = tomoq.query()
index = len(results) + 1
pixelsize = 1e-10 * apix * subbin
runname = params['volume']
shape = map((lambda x: x * subbin), params['shape'])
dimension = {'x': shape[2], 'y': shape[1], 'z': shape[0]}
subtomorundata = insertSubTomoRun(sessiondata, None, None, runname,
params['invert'], subbin)
return insertSubTomogram(fulltomogram, subtomorundata, None, 0,
dimension, path, name, index, pixelsize,
description)
