def setupTiltSeries(self):
tilts,self.ordered_imagelist,ordered_mrc_files,refindex = apTomo.orderImageList(self.orig_imagelist)
self.seriesname = apTomo.getFilename(self.tiltdatalist)
stackdir = self.params['tiltseriesdir']
stackname = self.seriesname+".st"
apTomo.writeTiltSeriesStack(stackdir,stackname,ordered_mrc_files,1e10*self.pixelsize)
apImod.writeRawtltFile(stackdir,self.seriesname,tilts)
def prepareTiltFile(sessionname, seriesname, tiltfilename, tiltseriesnumber, raw_path, frame_aligned_images, link=False, coarse=True):
'''
Creates tlt file from basic image information and copies raw images
'''
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata)
tiltdata = apTomo.getImageList([tiltseriesdata])
apDisplay.printMsg("getting imagelist")
frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata)
tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images)
if frame_aligned_images == "True": #Azimuth is only present in the non-frame aligned images
a,ordered_imagelist_for_azimuth,c,d,e = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned="False")
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
maxtilt = max([abs(tilts[0]),abs(tilts[-1])])
apDisplay.printMsg("highest tilt angle is %f" % maxtilt)
if coarse == "True":
if frame_aligned_images == "True": #Azimuth is only present in the non-frame aligned images
azimuth = apTomo.getAverageAzimuthFromSeries(ordered_imagelist_for_azimuth)
else:
azimuth = apTomo.getAverageAzimuthFromSeries(ordered_imagelist)
rawexists = apParam.createDirectory(raw_path)
apDisplay.printMsg("Copying raw images, y-flipping, normalizing, and converting images to float32 for Protomo...") #Linking removed because raw images need to be y-flipped for Protomo:(.
newfilenames, new_ordered_imagelist = apProTomo.getImageFiles(ordered_imagelist, raw_path, link=False, copy="True")
###create tilt file
#get image size from the first image
imagesizex = tiltdata[0]['image'].shape[1]
imagesizey = tiltdata[0]['image'].shape[0]
#shift half tilt series relative to eachother
#SS I'm arbitrarily making the bin parameter here 1 because it's not necessary to sample at this point
shifts = apTomo.getGlobalShift(ordered_imagelist, 1, refimg)
#OPTION: refinement might be more robust by doing one round of IMOD aligment to prealign images before doing protomo refine
origins = convertShiftsToOrigin(shifts, imagesizex, imagesizey)
writeTiltFile2(tiltfilename, seriesname, newfilenames, origins, tilts, azimuth, refimg)
return tilts, accumulated_dose_list, new_ordered_imagelist, maxtilt
def setupTiltSeries(self):
tilts, self.ordered_imagelist, ordered_mrc_files, refindex = apTomo.orderImageList(
self.orig_imagelist)
self.seriesname = apTomo.getFilename(self.tiltdatalist)
stackdir = self.params['tiltseriesdir']
stackname = self.seriesname + ".st"
apTomo.writeTiltSeriesStack(stackdir, stackname, ordered_mrc_files,
1e10 * self.pixelsize)
apImod.writeRawtltFile(stackdir, self.seriesname, tilts)
def doseCompensate(seriesname, rundir, sessionname, tiltseriesnumber, frame_aligned_images, raw_path, pixelsize, dose_presets, dose_a, dose_b, dose_c):
"""
Images will be lowpass filtered using equation (3) from Grant & Grigorieff, 2015.
No changes to the database are made. No backups are made.
"""
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata)
tiltdata = apTomo.getImageList([tiltseriesdata])
frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata)
tilts, ordered_imagelist, accumulated_dose_list, ordered_mrc_files, refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned="False")
if frame_aligned_images == "True": #For different image filenames
a, ordered_imagelist, c, d, e = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images)
newfilenames, new_ordered_imagelist = apProTomo.getImageFiles(ordered_imagelist, raw_path, link=False, copy=False)
if (dose_presets == "Light"):
dose_a = 0.245
dose_b = -1.6
dose_c = 12
elif (dose_presets == "Moderate"):
dose_a = 0.245
dose_b = -1.665
dose_c = 2.81
elif (dose_presets == "Heavy"):
dose_a = 0.245
dose_b = -1.4
dose_c = 2
apDisplay.printMsg('Dose compensating all tilt images with a=%s, b=%s, and c=%s...' % (dose_a, dose_b, dose_c))
for image, j in zip(new_ordered_imagelist, range(len(new_ordered_imagelist))):
lowpass = float(np.real(complex(dose_a/(accumulated_dose_list[j] - dose_c))**(1/dose_b))) #equation (3) from Grant & Grigorieff, 2015
if lowpass < 0.0:
lowpass = 0.0
im = mrc.read(image)
im = imagefilter.lowPassFilter(im, apix=pixelsize, radius=lowpass, msg=False)
im=imagenorm.normStdev(im)
mrc.write(im, image)
#Make plots
apProTomo2Aligner.makeDosePlots(rundir, seriesname, tilts, accumulated_dose_list, dose_a, dose_b, dose_c)
apDisplay.printMsg("Dose compensation finished for tilt-series #%s!" % tiltseriesnumber)
return
def start(self):
### some of this should go in preloop functions
###do queries
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata)
tiltseriessettings= apTomo.getTomographySettings(sessiondata,tiltseriesdata)
tiltdata=apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
print "ordering list"
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata)
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
###set up files
seriesname=self.params['seriesname']
print type(seriesname)
# param_out=seriesname+'.param'
###insert protomo run params
print "first insert"
protomodata = apProTomo.insertProtomoParams(seriesname)
print "second insert"
alignrun = apTomo.insertTomoAlignmentRun(sessiondata,tiltseriessettings,None,protomodata,None,1,self.params['runname'],self.params['rundir'],self.params['description'])
###insert protomo alignment
###hack to get around need to parse protomo param file
#should read imgref from tlt file
refineparamdict={'alismp':None,'alibox_x':None,'alibox_y':None,'cormod':None,'imgref':None}
###
self.params['goodcycle']=None
if self.params['goodrange'] is None:
self.params['goodstart']=1
self.params['goodend']=len(tilts)
alignerdata = apProTomo.insertAlignIteration(alignrun, protomodata, self.params, refineparamdict,ordered_imagelist[refimg])
# read tlt file
print "third insert"
alignmentdict, geometrydict, seriesname = apProTomo.parseTilt(self.params['tltfile'])
# insert geometry model
modeldata = apProTomo.insertModel2(alignerdata, geometrydict)
#insert image alignments
for i,imagedata in enumerate(ordered_imagelist):
#Caution...assumes ordered_imagelist is in same order as tlt file
apProTomo.insertTiltAlignment(alignerdata,imagedata,i,alignmentdict[i+1],center=None)
print "fourth insert"
apTomo.insertTiltsInAlignRun(alignrun, tiltseriesdata,tiltseriessettings,True)
def start(self):
### some of this should go in preloop functions
###do queries
os.chdir(self.params['rundir'])
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata)
tiltdata = apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata)
def start(self):
###do queries
sessiondata = apDatabase.getSessionDataFromSessionName(
self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(
self.params['tiltseries'], sessiondata)
tiltseriessettings = apTomo.getTomographySettings(
sessiondata, tiltseriesdata)
tiltdata = apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
print "ordering list"
tilts, ordered_imagelist, ordered_mrc_files, refimg = apTomo.orderImageList(
tiltdata)
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
###set up files
seriesname = 'series' + str(self.params['tiltseries'])
tiltfilename = seriesname + '.tlt'
param_out = seriesname + '.param'
maxtilt = max([abs(tilts[0]), abs(tilts[-1])])
apDisplay.printMsg("highest tilt angle is %f" % maxtilt)
self.params['cos_alpha'] = math.cos(maxtilt * math.pi / 180)
self.params['raw_path'] = os.path.join(self.params['rundir'], 'raw')
rawexists = apParam.createDirectory(self.params['raw_path'])
apDisplay.printMsg("copying raw images")
newfilenames = apProTomo.getImageFiles(ordered_imagelist,
self.params['raw_path'],
link=False)
#get alignment data
alignerdata = apTomo.getAlignerdata(self.params['alignerid'])
imgshape = apTomo.getTomoImageShape(ordered_imagelist[0])
imgcenter = {'x': self.imgshape[1] / 2, 'y': self.imgshape[0] / 2}
specimen_euler, azimuth, origins, rotations = apTomo.getAlignmentFromDB(
alignerdata, imgcenter)
#write protomo2 tilt file
outtltfile = 'series.tlt'
seriesname = 'series'
apProTomo.writeTiltFile2(outfilename, seriesname, specimen_eulers,
azimuth, referenceimage)
def start(self):
### some of this should go in preloop functions
###do queries
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata)
tiltdata=apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata)
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
###set up files
seriesname='series'+str(self.params['tiltseries'])
tiltfilename=seriesname+'.tlt'
param_out=seriesname+'.param'
maxtilt=max([abs(tilts[0]),abs(tilts[-1])])
apDisplay.printMsg("highest tilt angle is %f" % maxtilt)
self.params['cos_alpha']=math.cos(maxtilt*math.pi/180)
self.params['raw_path']=os.path.join(self.params['rundir'],'raw')
rawexists=apParam.createDirectory(self.params['raw_path'])
apDisplay.printMsg("copying raw images")
newfilenames=apProTomo.getImageFiles(ordered_imagelist,self.params['raw_path'], link=False)
###create tilt file
#get image size from the first image
imagesizex=tiltdata[0]['image'].shape[0]
imagesizey=tiltdata[0]['image'].shape[1]
#shift half tilt series relative to eachother
#SS I'm arbitrarily making the bin parameter here 1 because it's not necessary to sample at this point
shifts = apTomo.getGlobalShift(ordered_imagelist, 1, refimg)
#OPTION: refinement might be more robust by doing one round of IMOD aligment to prealign images before doing protomo refine
origins=apProTomo2Prep.convertShiftsToOrigin(shifts, imagesizex, imagesizey)
#determine azimuth
azimuth=apTomo.getAverageAzimuthFromSeries(ordered_imagelist)
apProTomo2Prep.writeTileFile2(tiltfilename, seriesname, newfilenames, origins, tilts, azimuth, refimg)
def start(self):
###do queries
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata)
tiltseriessettings= apTomo.getTomographySettings(sessiondata,tiltseriesdata)
tiltdata=apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
print "ordering list"
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata)
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
###set up files
seriesname='series'+str(self.params['tiltseries'])
tiltfilename=seriesname+'.tlt'
param_out=seriesname+'.param'
maxtilt=max([abs(tilts[0]),abs(tilts[-1])])
apDisplay.printMsg("highest tilt angle is %f" % maxtilt)
self.params['cos_alpha']=math.cos(maxtilt*math.pi/180)
self.params['raw_path']=os.path.join(self.params['rundir'],'raw')
rawexists=apParam.createDirectory(self.params['raw_path'])
apDisplay.printMsg("copying raw images")
newfilenames=apProTomo.getImageFiles(ordered_imagelist,self.params['raw_path'], link=False)
#get alignment data
alignerdata = apTomo.getAlignerdata(self.params['alignerid'])
imgshape = apTomo.getTomoImageShape(ordered_imagelist[0])
imgcenter = {'x':self.imgshape[1]/2,'y':self.imgshape[0]/2}
specimen_euler, azimuth, origins, rotations = apTomo.getAlignmentFromDB(alignerdata,imgcenter)
#write protomo2 tilt file
outtltfile='series.tlt'
seriesname='series'
apProTomo.writeTiltFile2(outfilename, seriesname, specimen_eulers, azimuth, referenceimage )
def start(self):
commit = self.params['commit']
tiltdatalist = apTomo.getTiltdataList(self.params['tiltseries'],self.params['othertiltseries'])
sessiondata = tiltdatalist[0]['session']
description = self.params['description']
alignsample = self.params['sample']
cycle = self.params['cycle']
alignmethod = self.params['alignmethod']
apDisplay.printMsg("getting imagelist")
imagelist = apTomo.getImageList(tiltdatalist)
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(imagelist)
# This parameter is needed for protomo, but not protomo2
if self.params['refimg']:
refimg = self.params['refimg']
if alignmethod != 'protomo2':
for file in ordered_mrc_files:
# protomo can not function with a negative origin
# protomo2 CAN function with negative origin
apImage.shiftMRCStartToZero(file)
apDisplay.printMsg("getting pixelsize")
pixelsize = apTomo.getTomoPixelSize(ordered_imagelist[refimg])
if pixelsize is None:
apDisplay.printError('Pixel Size not retrieved. Invalid tilt series for processing')
imgshape = apTomo.getTomoImageShape(ordered_imagelist[refimg])
corr_bin = apTomo.getCorrelatorBinning(imgshape)
center = {'x':imgshape[1]/2,'y':imgshape[0]/2}
default_azimuth = apTomo.getDefaultAzimuthFromLeginon(ordered_imagelist[refimg])
processdir = os.path.abspath(self.params['rundir'])
imodseriesname = apTomo.getFilename(tiltdatalist)
# protomo2 does not write out text files, intermediate info is in memory and the final result is binary
seriesname = 'tomo'+ imodseriesname
# Write tilt series stack images and tilt angles, not needed for protomo2
if alignmethod != 'protomo2':
stackdir = self.params['tiltseriesdir']
stackname = imodseriesname+".st"
apTomo.writeTiltSeriesStack(stackdir,stackname,ordered_mrc_files,1e10*pixelsize)
apImod.writeRawtltFile(stackdir,imodseriesname,tilts)
leginonxcorrlist = []
for tiltdata in tiltdatalist:
settingsdata = apTomo.getTomographySettings(sessiondata,tiltdata)
leginonxcorrlist.append(settingsdata)
# Run protomo2
if alignmethod == 'protomo2':
self.runProtomo2(sessiondata, processdir, seriesname, ordered_imagelist, refimg, center, corr_bin, commit, tilts)
# protomo2 does not need anything beyond this point, so exit
return
if cycle != 1 or alignmethod != 'protomo':
cycles=[cycle,]
else:
# also process and commit protomo cycle 0 if doing cycle 1
cycles=[0,1]
for cycle in cycles:
if alignmethod == 'protomo' or alignmethod == 'leginon':
self.params['aligndir'],self.params['imagedir'] = apProTomo.setProtomoDir(self.params['rundir'],cycle)
aligndir = self.params['aligndir']
# Link images into rundir/raw
rawimagenames = apProTomo.linkImageFiles(ordered_imagelist,self.params['imagedir'])
tltfile = os.path.join(aligndir,seriesname+'-%02d-itr.tlt' % (cycle,))
if cycle == 0:
# get initial shift alignment from leginon tiltcorrelator
# Assume all tiltdata have the same tomography settings
shifts = apTomo.getGlobalShift(ordered_imagelist, corr_bin, refimg)
tltparams = apProTomo.convertShiftsToParams(tilts,shifts,center,default_azimuth,rawimagenames)
apProTomo.writeTiltFile(tltfile,seriesname, tltparams[0], tltparams[1])
refineparamdict=apProTomo.createRefineDefaults(len(tilts),
os.path.join(processdir,'raw'),os.path.join(processdir,'out'))
refineparamdict = apProTomo.updateRefineParams(refineparamdict,imgshape,alignsample,100,refimg)
else:
lasttltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (cycle-1,))
if self.params['goodcycle']:
if (self.params['goodstart'] == 0 and self.params['goodend'] == len(ordered_imagelist) - 1):
# revert goodcycle related params since nothing will be reset
self.params['goodcycle'] = None
self.params['goodstart'] = None
self.params['goodend'] = None
if not self.params['goodcycle']:
#default uses last cycle
shutil.copy(lasttltfile,tltfile)
else:
if not (self.params['goodstart'] or self.params['goodend']):
shutil.copy(lasttltfile,tltfile)
else:
# Reset bad ending tilts before alignment cycle
goodtltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (self.params['goodcycle'],))
goodtltparams = apProTomo.parseTilt(goodtltfile)
lasttltparams = apProTomo.parseTilt(lasttltfile)
tltparams = apProTomo.resetTiltParams(lasttltparams,goodtltparams,self.params['goodstart'],self.params['goodend'])
apProTomo.writeTiltFile(tltfile,seriesname, tltparams[0], tltparams[1])
lastrefineparamfile = os.path.join(aligndir,seriesname+'-%02d.param' % (cycle-1,))
refineparamdict = apProTomo.parseRefineParamFile(lastrefineparamfile)
refineparamdict = apProTomo.updateRefineParams(refineparamdict,imgshape,alignsample,self.params['region'],refimg)
# Write param file in rundir/align
paramfilepath = os.path.join(seriesname+'-%02d.param' % (cycle))
fullparamfilepath = os.path.join(aligndir,paramfilepath)
apProTomo.writeRefineParamFile(refineparamdict,fullparamfilepath)
# run porjalign script
os.chdir(aligndir)
if cycle > 0:
self.runProjalign(paramfilepath)
else:
lasttltfile = os.path.join(aligndir,seriesname+'-%02d-itr.tlt' % (cycle,))
newtltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (cycle,))
shutil.copy(lasttltfile,newtltfile)
# convert to imod alignment
alignpathprefix = os.path.join(processdir, seriesname)
centertuple = (center['x'],center['y'])
apProTomo.convertGlobalTransformProtomoToImod(seriesname+'-%02d' %(cycle),imodseriesname, centertuple)
elif alignmethod == 'imod-shift':
# Correlation by Coarse correlation in IMOD
aligndir = processdir
imodxcorrdata = apImod.coarseAlignment(stackdir, processdir, imodseriesname, commit)
# Global Transformation
gtransforms = apImod.convertToGlobalAlignment(processdir, imodseriesname)
# Create Aligned Stack for record
bin = int(math.ceil(min(imgshape) / 512.0))
apImod.createAlignedStack(stackdir, aligndir, imodseriesname,bin)
if alignmethod == 'protomo' or alignmethod == 'leginon':
alifilename = imodseriesname+'-%02d.ali' % cycle
os.rename(imodseriesname+'.ali',alifilename)
else:
alifilename = imodseriesname+'.ali'
alifilepath = os.path.join(aligndir,alifilename)
# commit to database
if commit:
if alignmethod == 'protomo' or alignmethod == 'leginon':
# -- Commit Parameters --
protomodata = apProTomo.insertProtomoParams(seriesname)
alignrun = apTomo.insertTomoAlignmentRun(sessiondata,leginonxcorrlist[0],None,protomodata,None,1,self.params['runname'],self.params['rundir'],self.params['description'])
self.cycle_description = self.params['description']
self.params['cycle'] = cycle
if cycle == 0:
# temporarily change aligner description on the initial 0 cycle
self.params['description'] = 'leginon correlation results'
# insert sample and window size and all the other user defined params into ApProtomoRefinementParamsData
alignerdata = apProTomo.insertAlignIteration(alignrun, protomodata, self.params, refineparamdict,ordered_imagelist[refimg])
# -- Commit Results --
resulttltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (cycle,))
resulttltparams = apProTomo.parseTilt(resulttltfile)
if resulttltparams:
# commit the geometry parameters (psi, theta, phi, azimuth)
modeldata = apProTomo.insertModel(alignerdata, resulttltparams)
# insert results into ApProtomoAlignmentData (also used by imod) for each image
for i,imagedata in enumerate(ordered_imagelist):
apProTomo.insertTiltAlignment(alignerdata,imagedata,i,resulttltparams[0][i],center)
self.params['description'] = self.cycle_description
else:
alignrun = apTomo.insertTomoAlignmentRun(sessiondata,None,imodxcorrdata,None,None,1,self.params['runname'],self.params['rundir'],self.params['description'])
alignerdata = apTomo.insertAlignerParams(alignrun,self.params)
#results
prexgfile = os.path.join(aligndir,imodseriesname+'.prexg')
shifts = apImod.readShiftPrexgFile(aligndir, imodseriesname)
resulttltparams = apProTomo.convertShiftsToParams(tilts,shifts,center,default_azimuth)
if resulttltparams:
modeldata = apProTomo.insertModel(alignerdata, resulttltparams)
for i,imagedata in enumerate(ordered_imagelist):
apProTomo.insertTiltAlignment(alignerdata,imagedata,i,resulttltparams[0][i],center)
# multiple tilt series in one alignrun
for i in range(0,len(tiltdatalist)):
if i == 0:
primary = True
else:
primary = False
apTomo.insertTiltsInAlignRun(alignrun, tiltdatalist[i],leginonxcorrlist[i],primary)
apTomo.makeAlignStackMovie(alifilepath)
os.chdir(processdir)
def start(self):
# set local parameters
commit = self.params['commit']
tiltdatalist = apTomo.getTiltdataList(self.params['tiltseries'],
self.params['othertiltseries'])
sessiondata = tiltdatalist[0]['session']
description = self.params['description']
runname = self.params['runname']
alignmethod = self.params['alignmethod']
reconbin = int(self.params['reconbin'])
thickness_pixel = int(self.params['reconthickness'])
markersize_nm = int(self.params['markersize'])
markernumber = int(self.params['markernumber'])
apDisplay.printMsg("getting imagelist")
imagelist = apTomo.getImageList(tiltdatalist)
tilts, ordered_imagelist, ordered_mrc_files, refimg = apTomo.orderImageList(
imagelist)
apDisplay.printMsg("getting pixelsize")
pixelsize = apTomo.getTomoPixelSize(ordered_imagelist[refimg])
imgshape = apTomo.getTomoImageShape(ordered_imagelist[refimg])
#thickness_binnedpixel = int(thickness_nm * 1e-9 / (pixelsize * reconbin))
markersize_pixel = int(markersize_nm * 1e-9 / pixelsize)
processdir = os.path.abspath(self.params['rundir'])
imodseriesname = apTomo.getFilename(tiltdatalist)
seriesname = imodseriesname
# Write tilt series stack images and tilt angles
stackdir = self.params['tiltseriesdir']
stackname = imodseriesname + ".st"
apTomo.writeTiltSeriesStack(stackdir, stackname, ordered_mrc_files,
1e10 * pixelsize)
apRaptor.linkStToMrcExtension(stackdir, imodseriesname)
apImod.writeRawtltFile(stackdir, imodseriesname, tilts)
# Get Leginon tomography settings
leginontomosettingslist = []
for tiltdata in tiltdatalist:
settingsdata = apTomo.getTomographySettings(sessiondata, tiltdata)
leginontomosettingslist.append(settingsdata)
aligndir = processdir
# run the script and get alignment results when raptor can output alignment results in the future. raptoraligndata is None for now.
returncode, raptoraligndata, raptorfailed = apRaptor.alignAndRecon(
stackdir, stackname, processdir, markersize_pixel, reconbin,
thickness_pixel, markernumber, commit)
# Create Aligned Stack for record, not done in apRaptor yet, currently raptoraligndata is None
if not raptorfailed:
alifilename = imodseriesname + '.ali'
alifilepath = os.path.join(aligndir, 'align', alifilename)
print alifilepath
# commit to database
if commit:
# parameters
raptorparamsdata = apRaptor.insertRaptorParams(
markersize_nm, markernumber)
alignrun = apTomo.insertTomoAlignmentRun(
sessiondata, None, None, None, raptorparamsdata, 1,
self.params['runname'], self.params['rundir'],
self.params['description'], raptorfailed)
# to accomodate iterative alignment, one alignmentrun may have
# used the aligner several times, for this case a single
# aligner params data is inserted as in the case of Imod xcorr
alignerdata = apTomo.insertAlignerParams(alignrun, self.params)
#results
if raptoraligndata:
# if raptor has alignment result, it is converted to protomo
# format which is more parameterized and saved
prexgfile = os.path.join(aligndir, imodseriesname + '.prexg')
shifts = apImod.readShiftPrexgFile(aligndir, imodseriesname)
resulttltparams = apProTomo.convertShiftsToParams(
tilts, shifts, center)
if resulttltparams:
modeldata = apProTomo.insertModel(alignerdata,
resulttltparams)
for i, imagedata in enumerate(ordered_imagelist):
apProTomo.insertTiltAlignment(alignerdata, imagedata,
i, resulttltparams[0][i],
center)
# multiple tilt series in one alignrun
for i in range(0, len(tiltdatalist)):
if i == 0:
primary = True
else:
primary = False
# Record tilts in align run allows more than one tilt series to be
# used in one align run.
apTomo.insertTiltsInAlignRun(alignrun, tiltdatalist[i],
leginontomosettingslist[i],
primary)
if not raptorfailed:
apTomo.makeAlignStackMovie(alifilepath)
os.chdir(processdir)
# Full tomogram created with raptor is ???? handness?????
if not raptorfailed:
'''
voltransform = '????'
origtomopath = os.path.join(processdir, seriesname+"_full.rec")
currenttomopath = apImod.transformVolume(origtomopath,voltransform)
shutil.move(currenttomopath, origtomopath)
'''
zprojectfile = apImod.projectFullZ(processdir, runname,
seriesname, reconbin, False,
False)
try:
zimagedata = apTomo.uploadZProjection(
runname, imagelist[0], zprojectfile)
except:
zimagedata = None
fullrundata = apTomo.insertFullTomoRun(sessiondata, processdir,
runname, 'imod-wbp')
fulltomodata = apTomo.insertFullTomogram(
sessiondata, tiltdatalist[0], alignerdata, fullrundata,
runname, description, zimagedata, thickness_pixel,
reconbin)
# if raptor succeeded, upload data and parameters to database
session_time = sessiondata.timestamp
description = self.params['description']
raptordatabase = apRaptor.commitToJensenDatabase(
session_time, fulltomodata, stackdir, processdir,
stackname, description)
if raptordatabase == 0:
apDisplay.printMsg(
"RAPTOR and uploading to Jensen database done.")
else:
apDisplay.printWarning(
"Uploading to Jensen database failed.")
def start(self):
# set local parameters
commit = self.params['commit']
tiltdatalist = apTomo.getTiltdataList(self.params['tiltseries'],self.params['othertiltseries'])
sessiondata = tiltdatalist[0]['session']
description = self.params['description']
runname = self.params['runname']
alignmethod = self.params['alignmethod']
reconbin = int(self.params['reconbin'])
thickness_pixel = int(self.params['reconthickness'])
markersize_nm = int(self.params['markersize'])
markernumber = int(self.params['markernumber'])
apDisplay.printMsg("getting imagelist")
imagelist = apTomo.getImageList(tiltdatalist)
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(imagelist)
apDisplay.printMsg("getting pixelsize")
pixelsize = apTomo.getTomoPixelSize(ordered_imagelist[refimg])
imgshape = apTomo.getTomoImageShape(ordered_imagelist[refimg])
#thickness_binnedpixel = int(thickness_nm * 1e-9 / (pixelsize * reconbin))
markersize_pixel = int(markersize_nm * 1e-9 / pixelsize)
processdir = os.path.abspath(self.params['rundir'])
imodseriesname = apTomo.getFilename(tiltdatalist)
seriesname = imodseriesname
# Write tilt series stack images and tilt angles
stackdir = self.params['tiltseriesdir']
stackname = imodseriesname+".st"
apTomo.writeTiltSeriesStack(stackdir,stackname,ordered_mrc_files,1e10*pixelsize)
apRaptor.linkStToMrcExtension(stackdir,imodseriesname)
apImod.writeRawtltFile(stackdir,imodseriesname,tilts)
# Get Leginon tomography settings
leginontomosettingslist = []
for tiltdata in tiltdatalist:
settingsdata = apTomo.getTomographySettings(sessiondata,tiltdata)
leginontomosettingslist.append(settingsdata)
aligndir = processdir
# run the script and get alignment results when raptor can output alignment results in the future. raptoraligndata is None for now.
returncode, raptoraligndata, raptorfailed = apRaptor.alignAndRecon(stackdir, stackname, processdir, markersize_pixel, reconbin, thickness_pixel, markernumber, commit)
# Create Aligned Stack for record, not done in apRaptor yet, currently raptoraligndata is None
if not raptorfailed:
alifilename = imodseriesname+'.ali'
alifilepath = os.path.join(aligndir,'align',alifilename)
print alifilepath
# commit to database
if commit:
# parameters
raptorparamsdata = apRaptor.insertRaptorParams(markersize_nm,markernumber)
alignrun = apTomo.insertTomoAlignmentRun(sessiondata,None,None,None,raptorparamsdata,1,self.params['runname'],self.params['rundir'],self.params['description'],raptorfailed)
# to accomodate iterative alignment, one alignmentrun may have
# used the aligner several times, for this case a single
# aligner params data is inserted as in the case of Imod xcorr
alignerdata = apTomo.insertAlignerParams(alignrun,self.params)
#results
if raptoraligndata:
# if raptor has alignment result, it is converted to protomo
# format which is more parameterized and saved
prexgfile = os.path.join(aligndir,imodseriesname+'.prexg')
shifts = apImod.readShiftPrexgFile(aligndir, imodseriesname)
resulttltparams = apProTomo.convertShiftsToParams(tilts,shifts,center)
if resulttltparams:
modeldata = apProTomo.insertModel(alignerdata, resulttltparams)
for i,imagedata in enumerate(ordered_imagelist):
apProTomo.insertTiltAlignment(alignerdata,imagedata,i,resulttltparams[0][i],center)
# multiple tilt series in one alignrun
for i in range(0,len(tiltdatalist)):
if i == 0:
primary = True
else:
primary = False
# Record tilts in align run allows more than one tilt series to be
# used in one align run.
apTomo.insertTiltsInAlignRun(alignrun, tiltdatalist[i],leginontomosettingslist[i],primary)
if not raptorfailed:
apTomo.makeAlignStackMovie(alifilepath)
os.chdir(processdir)
# Full tomogram created with raptor is ???? handness?????
if not raptorfailed:
'''
voltransform = '????'
origtomopath = os.path.join(processdir, seriesname+"_full.rec")
currenttomopath = apImod.transformVolume(origtomopath,voltransform)
shutil.move(currenttomopath, origtomopath)
'''
zprojectfile = apImod.projectFullZ(processdir, runname, seriesname,reconbin,False,False)
try:
zimagedata = apTomo.uploadZProjection(runname,imagelist[0],zprojectfile)
except:
zimagedata = None
fullrundata = apTomo.insertFullTomoRun(sessiondata,processdir,runname,'imod-wbp')
fulltomodata = apTomo.insertFullTomogram(sessiondata,tiltdatalist[0],alignerdata,
fullrundata,runname,description,zimagedata,thickness_pixel,reconbin)
# if raptor succeeded, upload data and parameters to database
session_time = sessiondata.timestamp
description = self.params['description']
raptordatabase = apRaptor.commitToJensenDatabase(session_time, fulltomodata, stackdir, processdir, stackname, description)
if raptordatabase == 0:
apDisplay.printMsg("RAPTOR and uploading to Jensen database done.")
else:
apDisplay.printWarning("Uploading to Jensen database failed.")
#set up directories
print "Setting up directories"
rootdir=os.getcwd()
rawdir=os.path.join(rootdir, 'raw')
outdir=os.path.join(rootdir,'out')
if os.path.exists(rawdir) or os.path.exists(outdir) :
print "Warning, you must remove the raw, clean, align, and out directories before proceeding"
sys.exit()
else:
apParam.createDirectory(outdir,warning=False)
apParam.createDirectory(rawdir,warning=False)
tiltdata= getTiltSeriesFromId(inputparams['tiltid'])
tiltkeys,imgtree,mrcfiles,refindex = apTomo.orderImageList(tiltdata)
ptdict={}
zerotilts=[]
for n in range(len(imgtree)):
print "determining parameters for", imgtree[n]['filename']
imdict={}
imdictkey=n+1
#assign parameters to ptdict
tilt=imgtree[n]['scope']['stage position']['a']*180/math.pi
imdict['tilt']=tilt
origx=imgtree[n]['camera']['dimension']['x']/2
origy=imgtree[n]['camera']['dimension']['y']/2
imdict['rotation']=0
def start(self):
### some of this should go in preloop functions
###do queries
sessiondata = apDatabase.getSessionDataFromSessionName(
self.params['sessionname'])
self.sessiondata = sessiondata
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(
self.params['tiltseries'], sessiondata)
tiltseriessettings = apTomo.getTomographySettings(
sessiondata, tiltseriesdata)
tiltdata = apTomo.getImageList([tiltseriesdata])
description = self.params['description']
apDisplay.printMsg("getting imagelist")
print "ordering list"
tilts, ordered_imagelist, ordered_mrc_files, refimg = apTomo.orderImageList(
tiltdata)
#tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int
###set up files
seriesname = self.params['seriesname']
print type(seriesname)
# param_out=seriesname+'.param'
###insert protomo run params
print "first insert"
protomodata = apProTomo.insertProtomoParams(seriesname)
print "second insert"
alignrun = apTomo.insertTomoAlignmentRun(sessiondata,
tiltseriessettings, None,
protomodata, None, 1,
self.params['runname'],
self.params['rundir'],
self.params['description'])
###insert protomo alignment
###hack to get around need to parse protomo param file
#should read imgref from tlt file
refineparamdict = {
'alismp': None,
'alibox_x': None,
'alibox_y': None,
'cormod': None,
'imgref': None
}
###
self.params['goodcycle'] = None
if self.params['goodrange'] is None:
self.params['goodstart'] = 1
self.params['goodend'] = len(tilts)
alignerdata = apProTomo.insertAlignIteration(alignrun, protomodata,
self.params,
refineparamdict,
ordered_imagelist[refimg])
# read tlt file
print "third insert"
alignmentdict, geometrydict, seriesname = apProTomo.parseTilt(
self.params['tltfile'])
# insert geometry model
modeldata = apProTomo.insertModel2(alignerdata, geometrydict)
#insert image alignments
for i, imagedata in enumerate(ordered_imagelist):
#Caution...assumes ordered_imagelist is in same order as tlt file
apProTomo.insertTiltAlignment(alignerdata,
imagedata,
i,
alignmentdict[i + 1],
center=None)
print "fourth insert"
apTomo.insertTiltsInAlignRun(alignrun, tiltseriesdata,
tiltseriessettings, True)
#set up directories
print "Setting up directories"
rootdir = os.getcwd()
rawdir = os.path.join(rootdir, 'raw')
outdir = os.path.join(rootdir, 'out')
if os.path.exists(rawdir) or os.path.exists(outdir):
print "Warning, you must remove the raw, clean, align, and out directories before proceeding"
sys.exit()
else:
apParam.createDirectory(outdir, warning=False)
apParam.createDirectory(rawdir, warning=False)
tiltdata = getTiltSeriesFromId(inputparams['tiltid'])
tiltkeys, imgtree, mrcfiles, refindex = apTomo.orderImageList(tiltdata)
ptdict = {}
zerotilts = []
for n in range(len(imgtree)):
print "determining parameters for", imgtree[n]['filename']
imdict = {}
imdictkey = n + 1
#assign parameters to ptdict
tilt = imgtree[n]['scope']['stage position']['a'] * 180 / math.pi
imdict['tilt'] = tilt
origx = imgtree[n]['camera']['dimension']['x'] / 2
origy = imgtree[n]['camera']['dimension']['y'] / 2
imdict['rotation'] = 0
def ctfCorrect(seriesname, rundir, projectid, sessionname, tiltseriesnumber, tiltfilename, frame_aligned_images, pixelsize, DefocusTol, iWidth, amp_contrast):
"""
Leginondb will be queried to get the 'best' defocus estimate on a per-image basis.
Confident defoci will be gathered and unconfident defoci will be interpolated.
Images will be CTF corrected by phase flipping using ctfphaseflip from the IMOD package.
A plot of the defocus values will is made.
A CTF plot using the mean defocus is made.
"""
try:
apDisplay.printMsg('CTF correcting all tilt images using defocus values from Leginon database...')
os.chdir(rundir)
raw_path=rundir+'/raw/'
ctfdir='%s/ctf_correction/' % rundir
os.system("mkdir %s" % ctfdir)
defocus_file_full=ctfdir+seriesname+'_defocus.txt'
tilt_file_full=ctfdir+seriesname+'_tilts.txt'
image_list_full=ctfdir+seriesname+'_images.txt'
uncorrected_stack=ctfdir+'stack_uncorrected.mrc'
corrected_stack=ctfdir+'stack_corrected.mrc'
out_full=ctfdir+'out'
log_file_full=ctfdir+'ctf_correction.log'
project='ap'+projectid
sinedon.setConfig('appiondata', db=project)
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata)
tiltdata = apTomo.getImageList([tiltseriesdata])
frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata)
tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images)
cs = tiltdata[0]['scope']['tem']['cs']*1000
voltage = int(tiltdata[0]['scope']['high tension']/1000)
if os.path.isfile(ctfdir+'out/out01.mrc'): #Throw exception if already ctf corrected
sys.exit()
#Get tilt azimuth
cmd="awk '/TILT AZIMUTH/{print $3}' %s" % (tiltfilename)
proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
(tilt_azimuth, err) = proc.communicate()
tilt_azimuth=float(tilt_azimuth)
estimated_defocus=[]
for image in range(len(ordered_imagelist)):
imgctfdata=ctfdb.getBestCtfValue(ordered_imagelist[image], msg=False)
try:
if imgctfdata['resolution_50_percent'] < 100.0: #if there's a yellow ring in Appion, trust defocus estimation
estimated_defocus.append((imgctfdata['defocus1']+imgctfdata['defocus2'])*1000000000/2)
else: #Poorly estimated. Guess its value later
estimated_defocus.append(999999999)
except: #No data. Guess its value later
estimated_defocus.append(999999999)
#Find mean and stdev to prune out confident defocus values that are way off
defocus_stats_list=filter(lambda a: a != 999999999, estimated_defocus)
avg=np.array(defocus_stats_list).mean()
stdev=np.array(defocus_stats_list).std()
good_tilts=[]
good_defocus_list=[]
for tilt, defocus in zip(tilts, estimated_defocus):
if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev):
good_defocus_list.append(defocus)
good_tilts.append(tilt)
#Using a linear best fit because quadratic and cubic go off the rails. Estimation doesn't need to be extremely accurate anyways.
x=np.linspace(int(round(tilts[0])), int(round(tilts[len(tilts)-1])), 1000)
s=scipy.interpolate.UnivariateSpline(good_tilts,good_defocus_list,k=1)
y=s(x)
#Make defocus list with good values and interpolations for bad values
finished_defocus_list=[]
for tilt, defocus in zip(tilts, estimated_defocus):
if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev):
finished_defocus_list.append(int(round(defocus)))
else: #Interpolate
finished_defocus_list.append(int(round(y[int(round(tilt))])))
new_avg=np.array(finished_defocus_list).mean()
new_stdev=np.array(finished_defocus_list).std()
#Write defocus file, tilt file, and image list file for ctfphaseflip and newstack
f = open(defocus_file_full,'w')
f.write("%d\t%d\t%.2f\t%.2f\t%d\t2\n" % (1,1,tilts[0],tilts[0],finished_defocus_list[0]))
for i in range(1,len(tilts)):
f.write("%d\t%d\t%.2f\t%.2f\t%d\n" % (i+1,i+1,tilts[i],tilts[i],finished_defocus_list[i]))
f.close()
f = open(tilt_file_full,'w')
for tilt in tilts:
f.write("%.2f\n" % tilt)
f.close()
mrc_list=[]
presetname=tiltdata[0]['preset']['name']
for image in ordered_mrc_files:
mrcname=presetname+image.split(presetname)[-1]
mrc_list.append(raw_path+'/'+mrcname)
f = open(image_list_full,'w')
f.write("%d\n" % len(tilts))
for filename in mrc_list:
f.write(filename+'\n')
f.write("%d\n" % 0)
f.close()
#Rotate and pad images so that they are treated properly by ctfphaseflip.
apDisplay.printMsg("Preparing images for IMOD...")
for filename in mrc_list:
image=mrc.read(filename)
dimx=len(image[0])
dimy=len(image)
#First rotate 90 degrees in counter-clockwise direction. This makes it so positive angle images are higher defocused on the right side of the image
image=np.rot90(image, k=-1)
#Rotate image and write
image=imrotate(image, -tilt_azimuth, order=1) #Linear interpolation is fastest and there is barely a difference between linear and cubic
mrc.write(image, filename)
f = open(log_file_full,'w')
#Make stack for correction,phase flip, extract images, replace images
cmd1="newstack -fileinlist %s -output %s > %s" % (image_list_full, uncorrected_stack, log_file_full)
f.write("%s\n\n" % cmd1)
print cmd1
subprocess.check_call([cmd1], shell=True)
cmd2="ctfphaseflip -input %s -output %s -AngleFile %s -defFn %s -pixelSize %s -volt %s -DefocusTol %s -iWidth %s -SphericalAberration %s -AmplitudeContrast %s 2>&1 | tee %s" % (uncorrected_stack, corrected_stack, tilt_file_full, defocus_file_full, pixelsize/10, voltage, DefocusTol, iWidth, cs, amp_contrast, log_file_full)
f.write("\n\n%s\n\n" % cmd2)
print cmd2
subprocess.check_call([cmd2], shell=True)
cmd3="newstack -split 1 -append mrc %s %s >> %s" % (corrected_stack, out_full, log_file_full)
f.write("\n\n%s\n\n" % cmd3)
print cmd3
subprocess.check_call([cmd3], shell=True)
f.write("\n\n")
apDisplay.printMsg("Overwriting uncorrected raw images with CTF corrected images")
new_images=glob.glob(ctfdir+'out*mrc')
new_images.sort()
#Unrotate and unpad images
for filename in new_images:
image=mrc.read(filename)
image=imrotate(image, tilt_azimuth, order=1)
image=np.rot90(image, k=1)
big_dimx=len(image[0])
big_dimy=len(image)
cropx1=int((big_dimx-dimx)/2)
cropx2=int(dimx+(big_dimx-dimx)/2)
cropy1=int((big_dimy-dimy)/2)
cropy2=int(dimy+(big_dimy-dimy)/2)
image=image[cropy1:cropy2,cropx1:cropx2]
mrc.write(image, filename)
for i in range(len(new_images)):
cmd4="rm %s; ln %s %s" % (mrc_list[i], new_images[i], mrc_list[i])
f.write("%s\n" % cmd4)
os.system(cmd4)
#Make plots
apProTomo2Aligner.makeDefocusPlot(rundir, seriesname, defocus_file_full)
apProTomo2Aligner.makeCTFPlot(rundir, seriesname, defocus_file_full, voltage, cs)
cleanup="rm %s %s" % (uncorrected_stack, corrected_stack)
os.system(cleanup)
output1="%.2f%% of the images for tilt-series #%s had poor defocus estimates or fell outside of one standard deviation from the original mean." % (100*(len(estimated_defocus)-len(defocus_stats_list))/len(estimated_defocus), tiltseriesnumber)
output2="The defocus mean and standard deviation for tilt-series #%s after interpolating poor values is %.2f and %.2f microns, respectively." % (tiltseriesnumber, new_avg/1000, new_stdev/1000)
f.write("\n");f.write(output1);f.write("\n");f.write(output2);f.write("\n");f.close()
apDisplay.printMsg(output1)
apDisplay.printMsg(output2)
apDisplay.printMsg("CTF correction finished for tilt-series #%s!" % tiltseriesnumber)
except subprocess.CalledProcessError:
apDisplay.printError("An IMOD command failed, so CTF correction could not be completed. Make sure IMOD is in your $PATH.")
except SystemExit:
apDisplay.printWarning("It looks like you've already CTF corrected tilt-series #%s. Skipping CTF correction!" % tiltseriesnumber)
except:
apDisplay.printError("CTF correction could not be completed. Make sure IMOD, numpy, and scipy are in your $PATH. Make sure defocus has been estimated through Appion.\n")
