def insertAlignIteration(alignrundata, protomodata, params, refinedict,
refimagedata):
# protmoalign refinement cycle parameters
refineparamsq = appiondata.ApProtomoRefinementParamsData()
refineparamsq['protomo'] = protomodata
refineparamsq['cycle'] = params['cycle']
refineparamsq['alismp'] = refinedict['alismp']
refineparamsq['alibox'] = {
'x': refinedict['alibox_x'],
'y': refinedict['alibox_y']
}
refineparamsq['cormod'] = refinedict['cormod']
refineparamsq['imgref'] = refinedict['imgref']
refineparamsq['reference'] = refimagedata
refineparamsdata = apTomo.publish(refineparamsq)
# good cycle used for reset tlt params
if params['goodcycle'] is None:
goodrefineparamsdata = None
else:
goodq = appiondata.ApProtomoRefinementParamsData(
protomo=protomodata, cycle=params['goodcycle'])
results = goodq.query(results=1)
if results:
goodrefineparamsdata = results[0]
else:
goodrefineparamsdata = None
# protomoaligner parameters
alignerdata = apTomo.insertAlignerParams(alignrundata, params, protomodata,
refineparamsdata,
goodrefineparamsdata,
refimagedata)
return alignerdata
def insertAlignIteration(alignrundata, protomodata, params, refinedict,refimagedata):
# protmoalign refinement cycle parameters
refineparamsq = appiondata.ApProtomoRefinementParamsData()
refineparamsq['protomo'] = protomodata
refineparamsq['cycle'] = params['cycle']
refineparamsq['alismp'] = refinedict['alismp']
refineparamsq['alibox'] = {'x':refinedict['alibox_x'],'y':refinedict['alibox_y']}
refineparamsq['cormod'] = refinedict['cormod']
refineparamsq['imgref'] = refinedict['imgref']
refineparamsq['reference'] = refimagedata
refineparamsdata = apTomo.publish(refineparamsq)
# good cycle used for reset tlt params
if params['goodcycle'] is None:
goodrefineparamsdata = None
else:
goodq = appiondata.ApProtomoRefinementParamsData(protomo=protomodata,cycle=params['goodcycle'])
results = goodq.query(results=1)
if results:
goodrefineparamsdata = results[0]
else:
goodrefineparamsdata = None
# protomoaligner parameters
alignerdata = apTomo.insertAlignerParams(alignrundata,params,protomodata,refineparamsdata,goodrefineparamsdata,refimagedata)
return alignerdata
def start(self):
commit = self.params['commit']
tiltdatalist = apTomo.getTiltdataList(self.params['tiltseries'],self.params['othertiltseries'])
sessiondata = tiltdatalist[0]['session']
description = self.params['description']
alignsample = self.params['sample']
cycle = self.params['cycle']
alignmethod = self.params['alignmethod']
apDisplay.printMsg("getting imagelist")
imagelist = apTomo.getImageList(tiltdatalist)
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(imagelist)
# This parameter is needed for protomo, but not protomo2
if self.params['refimg']:
refimg = self.params['refimg']
if alignmethod != 'protomo2':
for file in ordered_mrc_files:
# protomo can not function with a negative origin
# protomo2 CAN function with negative origin
apImage.shiftMRCStartToZero(file)
apDisplay.printMsg("getting pixelsize")
pixelsize = apTomo.getTomoPixelSize(ordered_imagelist[refimg])
if pixelsize is None:
apDisplay.printError('Pixel Size not retrieved. Invalid tilt series for processing')
imgshape = apTomo.getTomoImageShape(ordered_imagelist[refimg])
corr_bin = apTomo.getCorrelatorBinning(imgshape)
center = {'x':imgshape[1]/2,'y':imgshape[0]/2}
default_azimuth = apTomo.getDefaultAzimuthFromLeginon(ordered_imagelist[refimg])
processdir = os.path.abspath(self.params['rundir'])
imodseriesname = apTomo.getFilename(tiltdatalist)
# protomo2 does not write out text files, intermediate info is in memory and the final result is binary
seriesname = 'tomo'+ imodseriesname
# Write tilt series stack images and tilt angles, not needed for protomo2
if alignmethod != 'protomo2':
stackdir = self.params['tiltseriesdir']
stackname = imodseriesname+".st"
apTomo.writeTiltSeriesStack(stackdir,stackname,ordered_mrc_files,1e10*pixelsize)
apImod.writeRawtltFile(stackdir,imodseriesname,tilts)
leginonxcorrlist = []
for tiltdata in tiltdatalist:
settingsdata = apTomo.getTomographySettings(sessiondata,tiltdata)
leginonxcorrlist.append(settingsdata)
# Run protomo2
if alignmethod == 'protomo2':
self.runProtomo2(sessiondata, processdir, seriesname, ordered_imagelist, refimg, center, corr_bin, commit, tilts)
# protomo2 does not need anything beyond this point, so exit
return
if cycle != 1 or alignmethod != 'protomo':
cycles=[cycle,]
else:
# also process and commit protomo cycle 0 if doing cycle 1
cycles=[0,1]
for cycle in cycles:
if alignmethod == 'protomo' or alignmethod == 'leginon':
self.params['aligndir'],self.params['imagedir'] = apProTomo.setProtomoDir(self.params['rundir'],cycle)
aligndir = self.params['aligndir']
# Link images into rundir/raw
rawimagenames = apProTomo.linkImageFiles(ordered_imagelist,self.params['imagedir'])
tltfile = os.path.join(aligndir,seriesname+'-%02d-itr.tlt' % (cycle,))
if cycle == 0:
# get initial shift alignment from leginon tiltcorrelator
# Assume all tiltdata have the same tomography settings
shifts = apTomo.getGlobalShift(ordered_imagelist, corr_bin, refimg)
tltparams = apProTomo.convertShiftsToParams(tilts,shifts,center,default_azimuth,rawimagenames)
apProTomo.writeTiltFile(tltfile,seriesname, tltparams[0], tltparams[1])
refineparamdict=apProTomo.createRefineDefaults(len(tilts),
os.path.join(processdir,'raw'),os.path.join(processdir,'out'))
refineparamdict = apProTomo.updateRefineParams(refineparamdict,imgshape,alignsample,100,refimg)
else:
lasttltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (cycle-1,))
if self.params['goodcycle']:
if (self.params['goodstart'] == 0 and self.params['goodend'] == len(ordered_imagelist) - 1):
# revert goodcycle related params since nothing will be reset
self.params['goodcycle'] = None
self.params['goodstart'] = None
self.params['goodend'] = None
if not self.params['goodcycle']:
#default uses last cycle
shutil.copy(lasttltfile,tltfile)
else:
if not (self.params['goodstart'] or self.params['goodend']):
shutil.copy(lasttltfile,tltfile)
else:
# Reset bad ending tilts before alignment cycle
goodtltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (self.params['goodcycle'],))
goodtltparams = apProTomo.parseTilt(goodtltfile)
lasttltparams = apProTomo.parseTilt(lasttltfile)
tltparams = apProTomo.resetTiltParams(lasttltparams,goodtltparams,self.params['goodstart'],self.params['goodend'])
apProTomo.writeTiltFile(tltfile,seriesname, tltparams[0], tltparams[1])
lastrefineparamfile = os.path.join(aligndir,seriesname+'-%02d.param' % (cycle-1,))
refineparamdict = apProTomo.parseRefineParamFile(lastrefineparamfile)
refineparamdict = apProTomo.updateRefineParams(refineparamdict,imgshape,alignsample,self.params['region'],refimg)
# Write param file in rundir/align
paramfilepath = os.path.join(seriesname+'-%02d.param' % (cycle))
fullparamfilepath = os.path.join(aligndir,paramfilepath)
apProTomo.writeRefineParamFile(refineparamdict,fullparamfilepath)
# run porjalign script
os.chdir(aligndir)
if cycle > 0:
self.runProjalign(paramfilepath)
else:
lasttltfile = os.path.join(aligndir,seriesname+'-%02d-itr.tlt' % (cycle,))
newtltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (cycle,))
shutil.copy(lasttltfile,newtltfile)
# convert to imod alignment
alignpathprefix = os.path.join(processdir, seriesname)
centertuple = (center['x'],center['y'])
apProTomo.convertGlobalTransformProtomoToImod(seriesname+'-%02d' %(cycle),imodseriesname, centertuple)
elif alignmethod == 'imod-shift':
# Correlation by Coarse correlation in IMOD
aligndir = processdir
imodxcorrdata = apImod.coarseAlignment(stackdir, processdir, imodseriesname, commit)
# Global Transformation
gtransforms = apImod.convertToGlobalAlignment(processdir, imodseriesname)
# Create Aligned Stack for record
bin = int(math.ceil(min(imgshape) / 512.0))
apImod.createAlignedStack(stackdir, aligndir, imodseriesname,bin)
if alignmethod == 'protomo' or alignmethod == 'leginon':
alifilename = imodseriesname+'-%02d.ali' % cycle
os.rename(imodseriesname+'.ali',alifilename)
else:
alifilename = imodseriesname+'.ali'
alifilepath = os.path.join(aligndir,alifilename)
# commit to database
if commit:
if alignmethod == 'protomo' or alignmethod == 'leginon':
# -- Commit Parameters --
protomodata = apProTomo.insertProtomoParams(seriesname)
alignrun = apTomo.insertTomoAlignmentRun(sessiondata,leginonxcorrlist[0],None,protomodata,None,1,self.params['runname'],self.params['rundir'],self.params['description'])
self.cycle_description = self.params['description']
self.params['cycle'] = cycle
if cycle == 0:
# temporarily change aligner description on the initial 0 cycle
self.params['description'] = 'leginon correlation results'
# insert sample and window size and all the other user defined params into ApProtomoRefinementParamsData
alignerdata = apProTomo.insertAlignIteration(alignrun, protomodata, self.params, refineparamdict,ordered_imagelist[refimg])
# -- Commit Results --
resulttltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (cycle,))
resulttltparams = apProTomo.parseTilt(resulttltfile)
if resulttltparams:
# commit the geometry parameters (psi, theta, phi, azimuth)
modeldata = apProTomo.insertModel(alignerdata, resulttltparams)
# insert results into ApProtomoAlignmentData (also used by imod) for each image
for i,imagedata in enumerate(ordered_imagelist):
apProTomo.insertTiltAlignment(alignerdata,imagedata,i,resulttltparams[0][i],center)
self.params['description'] = self.cycle_description
else:
alignrun = apTomo.insertTomoAlignmentRun(sessiondata,None,imodxcorrdata,None,None,1,self.params['runname'],self.params['rundir'],self.params['description'])
alignerdata = apTomo.insertAlignerParams(alignrun,self.params)
#results
prexgfile = os.path.join(aligndir,imodseriesname+'.prexg')
shifts = apImod.readShiftPrexgFile(aligndir, imodseriesname)
resulttltparams = apProTomo.convertShiftsToParams(tilts,shifts,center,default_azimuth)
if resulttltparams:
modeldata = apProTomo.insertModel(alignerdata, resulttltparams)
for i,imagedata in enumerate(ordered_imagelist):
apProTomo.insertTiltAlignment(alignerdata,imagedata,i,resulttltparams[0][i],center)
# multiple tilt series in one alignrun
for i in range(0,len(tiltdatalist)):
if i == 0:
primary = True
else:
primary = False
apTomo.insertTiltsInAlignRun(alignrun, tiltdatalist[i],leginonxcorrlist[i],primary)
apTomo.makeAlignStackMovie(alifilepath)
os.chdir(processdir)
def start(self):
# set local parameters
commit = self.params['commit']
tiltdatalist = apTomo.getTiltdataList(self.params['tiltseries'],
self.params['othertiltseries'])
sessiondata = tiltdatalist[0]['session']
description = self.params['description']
runname = self.params['runname']
alignmethod = self.params['alignmethod']
reconbin = int(self.params['reconbin'])
thickness_pixel = int(self.params['reconthickness'])
markersize_nm = int(self.params['markersize'])
markernumber = int(self.params['markernumber'])
apDisplay.printMsg("getting imagelist")
imagelist = apTomo.getImageList(tiltdatalist)
tilts, ordered_imagelist, ordered_mrc_files, refimg = apTomo.orderImageList(
imagelist)
apDisplay.printMsg("getting pixelsize")
pixelsize = apTomo.getTomoPixelSize(ordered_imagelist[refimg])
imgshape = apTomo.getTomoImageShape(ordered_imagelist[refimg])
#thickness_binnedpixel = int(thickness_nm * 1e-9 / (pixelsize * reconbin))
markersize_pixel = int(markersize_nm * 1e-9 / pixelsize)
processdir = os.path.abspath(self.params['rundir'])
imodseriesname = apTomo.getFilename(tiltdatalist)
seriesname = imodseriesname
# Write tilt series stack images and tilt angles
stackdir = self.params['tiltseriesdir']
stackname = imodseriesname + ".st"
apTomo.writeTiltSeriesStack(stackdir, stackname, ordered_mrc_files,
1e10 * pixelsize)
apRaptor.linkStToMrcExtension(stackdir, imodseriesname)
apImod.writeRawtltFile(stackdir, imodseriesname, tilts)
# Get Leginon tomography settings
leginontomosettingslist = []
for tiltdata in tiltdatalist:
settingsdata = apTomo.getTomographySettings(sessiondata, tiltdata)
leginontomosettingslist.append(settingsdata)
aligndir = processdir
# run the script and get alignment results when raptor can output alignment results in the future. raptoraligndata is None for now.
returncode, raptoraligndata, raptorfailed = apRaptor.alignAndRecon(
stackdir, stackname, processdir, markersize_pixel, reconbin,
thickness_pixel, markernumber, commit)
# Create Aligned Stack for record, not done in apRaptor yet, currently raptoraligndata is None
if not raptorfailed:
alifilename = imodseriesname + '.ali'
alifilepath = os.path.join(aligndir, 'align', alifilename)
print alifilepath
# commit to database
if commit:
# parameters
raptorparamsdata = apRaptor.insertRaptorParams(
markersize_nm, markernumber)
alignrun = apTomo.insertTomoAlignmentRun(
sessiondata, None, None, None, raptorparamsdata, 1,
self.params['runname'], self.params['rundir'],
self.params['description'], raptorfailed)
# to accomodate iterative alignment, one alignmentrun may have
# used the aligner several times, for this case a single
# aligner params data is inserted as in the case of Imod xcorr
alignerdata = apTomo.insertAlignerParams(alignrun, self.params)
#results
if raptoraligndata:
# if raptor has alignment result, it is converted to protomo
# format which is more parameterized and saved
prexgfile = os.path.join(aligndir, imodseriesname + '.prexg')
shifts = apImod.readShiftPrexgFile(aligndir, imodseriesname)
resulttltparams = apProTomo.convertShiftsToParams(
tilts, shifts, center)
if resulttltparams:
modeldata = apProTomo.insertModel(alignerdata,
resulttltparams)
for i, imagedata in enumerate(ordered_imagelist):
apProTomo.insertTiltAlignment(alignerdata, imagedata,
i, resulttltparams[0][i],
center)
# multiple tilt series in one alignrun
for i in range(0, len(tiltdatalist)):
if i == 0:
primary = True
else:
primary = False
# Record tilts in align run allows more than one tilt series to be
# used in one align run.
apTomo.insertTiltsInAlignRun(alignrun, tiltdatalist[i],
leginontomosettingslist[i],
primary)
if not raptorfailed:
apTomo.makeAlignStackMovie(alifilepath)
os.chdir(processdir)
# Full tomogram created with raptor is ???? handness?????
if not raptorfailed:
'''
voltransform = '????'
origtomopath = os.path.join(processdir, seriesname+"_full.rec")
currenttomopath = apImod.transformVolume(origtomopath,voltransform)
shutil.move(currenttomopath, origtomopath)
'''
zprojectfile = apImod.projectFullZ(processdir, runname,
seriesname, reconbin, False,
False)
try:
zimagedata = apTomo.uploadZProjection(
runname, imagelist[0], zprojectfile)
except:
zimagedata = None
fullrundata = apTomo.insertFullTomoRun(sessiondata, processdir,
runname, 'imod-wbp')
fulltomodata = apTomo.insertFullTomogram(
sessiondata, tiltdatalist[0], alignerdata, fullrundata,
runname, description, zimagedata, thickness_pixel,
reconbin)
# if raptor succeeded, upload data and parameters to database
session_time = sessiondata.timestamp
description = self.params['description']
raptordatabase = apRaptor.commitToJensenDatabase(
session_time, fulltomodata, stackdir, processdir,
stackname, description)
if raptordatabase == 0:
apDisplay.printMsg(
"RAPTOR and uploading to Jensen database done.")
else:
apDisplay.printWarning(
"Uploading to Jensen database failed.")
def start(self):
# set local parameters
commit = self.params['commit']
tiltdatalist = apTomo.getTiltdataList(self.params['tiltseries'],self.params['othertiltseries'])
sessiondata = tiltdatalist[0]['session']
description = self.params['description']
runname = self.params['runname']
alignmethod = self.params['alignmethod']
reconbin = int(self.params['reconbin'])
thickness_pixel = int(self.params['reconthickness'])
markersize_nm = int(self.params['markersize'])
markernumber = int(self.params['markernumber'])
apDisplay.printMsg("getting imagelist")
imagelist = apTomo.getImageList(tiltdatalist)
tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(imagelist)
apDisplay.printMsg("getting pixelsize")
pixelsize = apTomo.getTomoPixelSize(ordered_imagelist[refimg])
imgshape = apTomo.getTomoImageShape(ordered_imagelist[refimg])
#thickness_binnedpixel = int(thickness_nm * 1e-9 / (pixelsize * reconbin))
markersize_pixel = int(markersize_nm * 1e-9 / pixelsize)
processdir = os.path.abspath(self.params['rundir'])
imodseriesname = apTomo.getFilename(tiltdatalist)
seriesname = imodseriesname
# Write tilt series stack images and tilt angles
stackdir = self.params['tiltseriesdir']
stackname = imodseriesname+".st"
apTomo.writeTiltSeriesStack(stackdir,stackname,ordered_mrc_files,1e10*pixelsize)
apRaptor.linkStToMrcExtension(stackdir,imodseriesname)
apImod.writeRawtltFile(stackdir,imodseriesname,tilts)
# Get Leginon tomography settings
leginontomosettingslist = []
for tiltdata in tiltdatalist:
settingsdata = apTomo.getTomographySettings(sessiondata,tiltdata)
leginontomosettingslist.append(settingsdata)
aligndir = processdir
# run the script and get alignment results when raptor can output alignment results in the future. raptoraligndata is None for now.
returncode, raptoraligndata, raptorfailed = apRaptor.alignAndRecon(stackdir, stackname, processdir, markersize_pixel, reconbin, thickness_pixel, markernumber, commit)
# Create Aligned Stack for record, not done in apRaptor yet, currently raptoraligndata is None
if not raptorfailed:
alifilename = imodseriesname+'.ali'
alifilepath = os.path.join(aligndir,'align',alifilename)
print alifilepath
# commit to database
if commit:
# parameters
raptorparamsdata = apRaptor.insertRaptorParams(markersize_nm,markernumber)
alignrun = apTomo.insertTomoAlignmentRun(sessiondata,None,None,None,raptorparamsdata,1,self.params['runname'],self.params['rundir'],self.params['description'],raptorfailed)
# to accomodate iterative alignment, one alignmentrun may have
# used the aligner several times, for this case a single
# aligner params data is inserted as in the case of Imod xcorr
alignerdata = apTomo.insertAlignerParams(alignrun,self.params)
#results
if raptoraligndata:
# if raptor has alignment result, it is converted to protomo
# format which is more parameterized and saved
prexgfile = os.path.join(aligndir,imodseriesname+'.prexg')
shifts = apImod.readShiftPrexgFile(aligndir, imodseriesname)
resulttltparams = apProTomo.convertShiftsToParams(tilts,shifts,center)
if resulttltparams:
modeldata = apProTomo.insertModel(alignerdata, resulttltparams)
for i,imagedata in enumerate(ordered_imagelist):
apProTomo.insertTiltAlignment(alignerdata,imagedata,i,resulttltparams[0][i],center)
# multiple tilt series in one alignrun
for i in range(0,len(tiltdatalist)):
if i == 0:
primary = True
else:
primary = False
# Record tilts in align run allows more than one tilt series to be
# used in one align run.
apTomo.insertTiltsInAlignRun(alignrun, tiltdatalist[i],leginontomosettingslist[i],primary)
if not raptorfailed:
apTomo.makeAlignStackMovie(alifilepath)
os.chdir(processdir)
# Full tomogram created with raptor is ???? handness?????
if not raptorfailed:
'''
voltransform = '????'
origtomopath = os.path.join(processdir, seriesname+"_full.rec")
currenttomopath = apImod.transformVolume(origtomopath,voltransform)
shutil.move(currenttomopath, origtomopath)
'''
zprojectfile = apImod.projectFullZ(processdir, runname, seriesname,reconbin,False,False)
try:
zimagedata = apTomo.uploadZProjection(runname,imagelist[0],zprojectfile)
except:
zimagedata = None
fullrundata = apTomo.insertFullTomoRun(sessiondata,processdir,runname,'imod-wbp')
fulltomodata = apTomo.insertFullTomogram(sessiondata,tiltdatalist[0],alignerdata,
fullrundata,runname,description,zimagedata,thickness_pixel,reconbin)
# if raptor succeeded, upload data and parameters to database
session_time = sessiondata.timestamp
description = self.params['description']
raptordatabase = apRaptor.commitToJensenDatabase(session_time, fulltomodata, stackdir, processdir, stackname, description)
if raptordatabase == 0:
apDisplay.printMsg("RAPTOR and uploading to Jensen database done.")
else:
apDisplay.printWarning("Uploading to Jensen database failed.")