def setupImageInfo(self): apDisplay.printMsg("getting imagelist") self.orig_imagelist = apTomo.getImageList(self.tiltdatalist) apDisplay.printMsg("getting pixelsize") self.zerotiltimage = self.orig_imagelist[0] self.pixelsize = apTomo.getTomoPixelSize(self.zerotiltimage) apDisplay.printMsg("getting image shape and center") self.imgshape = apTomo.getTomoImageShape(self.zerotiltimage) self.imgcenter = {'x':self.imgshape[1]/2,'y':self.imgshape[0]/2}
def setupImageInfo(self): apDisplay.printMsg("getting imagelist") self.orig_imagelist = apTomo.getImageList(self.tiltdatalist) apDisplay.printMsg("getting pixelsize") self.zerotiltimage = self.orig_imagelist[0] self.pixelsize = apTomo.getTomoPixelSize(self.zerotiltimage) apDisplay.printMsg("getting image shape and center") self.imgshape = apTomo.getTomoImageShape(self.zerotiltimage) self.imgcenter = {'x': self.imgshape[1] / 2, 'y': self.imgshape[0] / 2}
def start(self): ### some of this should go in preloop functions ###do queries sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata) tiltseriessettings= apTomo.getTomographySettings(sessiondata,tiltseriesdata) tiltdata=apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") print "ordering list" tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata) #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int ###set up files seriesname=self.params['seriesname'] print type(seriesname) # param_out=seriesname+'.param' ###insert protomo run params print "first insert" protomodata = apProTomo.insertProtomoParams(seriesname) print "second insert" alignrun = apTomo.insertTomoAlignmentRun(sessiondata,tiltseriessettings,None,protomodata,None,1,self.params['runname'],self.params['rundir'],self.params['description']) ###insert protomo alignment ###hack to get around need to parse protomo param file #should read imgref from tlt file refineparamdict={'alismp':None,'alibox_x':None,'alibox_y':None,'cormod':None,'imgref':None} ### self.params['goodcycle']=None if self.params['goodrange'] is None: self.params['goodstart']=1 self.params['goodend']=len(tilts) alignerdata = apProTomo.insertAlignIteration(alignrun, protomodata, self.params, refineparamdict,ordered_imagelist[refimg]) # read tlt file print "third insert" alignmentdict, geometrydict, seriesname = apProTomo.parseTilt(self.params['tltfile']) # insert geometry model modeldata = apProTomo.insertModel2(alignerdata, geometrydict) #insert image alignments for i,imagedata in enumerate(ordered_imagelist): #Caution...assumes ordered_imagelist is in same order as tlt file apProTomo.insertTiltAlignment(alignerdata,imagedata,i,alignmentdict[i+1],center=None) print "fourth insert" apTomo.insertTiltsInAlignRun(alignrun, tiltseriesdata,tiltseriessettings,True)
def start(self): ### some of this should go in preloop functions ###do queries os.chdir(self.params['rundir']) sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata) tiltdata = apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata)
def start(self): ###do queries sessiondata = apDatabase.getSessionDataFromSessionName( self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId( self.params['tiltseries'], sessiondata) tiltseriessettings = apTomo.getTomographySettings( sessiondata, tiltseriesdata) tiltdata = apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") print "ordering list" tilts, ordered_imagelist, ordered_mrc_files, refimg = apTomo.orderImageList( tiltdata) #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int ###set up files seriesname = 'series' + str(self.params['tiltseries']) tiltfilename = seriesname + '.tlt' param_out = seriesname + '.param' maxtilt = max([abs(tilts[0]), abs(tilts[-1])]) apDisplay.printMsg("highest tilt angle is %f" % maxtilt) self.params['cos_alpha'] = math.cos(maxtilt * math.pi / 180) self.params['raw_path'] = os.path.join(self.params['rundir'], 'raw') rawexists = apParam.createDirectory(self.params['raw_path']) apDisplay.printMsg("copying raw images") newfilenames = apProTomo.getImageFiles(ordered_imagelist, self.params['raw_path'], link=False) #get alignment data alignerdata = apTomo.getAlignerdata(self.params['alignerid']) imgshape = apTomo.getTomoImageShape(ordered_imagelist[0]) imgcenter = {'x': self.imgshape[1] / 2, 'y': self.imgshape[0] / 2} specimen_euler, azimuth, origins, rotations = apTomo.getAlignmentFromDB( alignerdata, imgcenter) #write protomo2 tilt file outtltfile = 'series.tlt' seriesname = 'series' apProTomo.writeTiltFile2(outfilename, seriesname, specimen_eulers, azimuth, referenceimage)
def start(self): ### some of this should go in preloop functions ###do queries sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata) tiltdata=apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata) #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int ###set up files seriesname='series'+str(self.params['tiltseries']) tiltfilename=seriesname+'.tlt' param_out=seriesname+'.param' maxtilt=max([abs(tilts[0]),abs(tilts[-1])]) apDisplay.printMsg("highest tilt angle is %f" % maxtilt) self.params['cos_alpha']=math.cos(maxtilt*math.pi/180) self.params['raw_path']=os.path.join(self.params['rundir'],'raw') rawexists=apParam.createDirectory(self.params['raw_path']) apDisplay.printMsg("copying raw images") newfilenames=apProTomo.getImageFiles(ordered_imagelist,self.params['raw_path'], link=False) ###create tilt file #get image size from the first image imagesizex=tiltdata[0]['image'].shape[0] imagesizey=tiltdata[0]['image'].shape[1] #shift half tilt series relative to eachother #SS I'm arbitrarily making the bin parameter here 1 because it's not necessary to sample at this point shifts = apTomo.getGlobalShift(ordered_imagelist, 1, refimg) #OPTION: refinement might be more robust by doing one round of IMOD aligment to prealign images before doing protomo refine origins=apProTomo2Prep.convertShiftsToOrigin(shifts, imagesizex, imagesizey) #determine azimuth azimuth=apTomo.getAverageAzimuthFromSeries(ordered_imagelist) apProTomo2Prep.writeTileFile2(tiltfilename, seriesname, newfilenames, origins, tilts, azimuth, refimg)
def prepareTiltFile(sessionname, seriesname, tiltfilename, tiltseriesnumber, raw_path, frame_aligned_images, link=False, coarse=True): ''' Creates tlt file from basic image information and copies raw images ''' sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata) tiltdata = apTomo.getImageList([tiltseriesdata]) apDisplay.printMsg("getting imagelist") frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata) tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images) if frame_aligned_images == "True": #Azimuth is only present in the non-frame aligned images a,ordered_imagelist_for_azimuth,c,d,e = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned="False") #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int maxtilt = max([abs(tilts[0]),abs(tilts[-1])]) apDisplay.printMsg("highest tilt angle is %f" % maxtilt) if coarse == "True": if frame_aligned_images == "True": #Azimuth is only present in the non-frame aligned images azimuth = apTomo.getAverageAzimuthFromSeries(ordered_imagelist_for_azimuth) else: azimuth = apTomo.getAverageAzimuthFromSeries(ordered_imagelist) rawexists = apParam.createDirectory(raw_path) apDisplay.printMsg("Copying raw images, y-flipping, normalizing, and converting images to float32 for Protomo...") #Linking removed because raw images need to be y-flipped for Protomo:(. newfilenames, new_ordered_imagelist = apProTomo.getImageFiles(ordered_imagelist, raw_path, link=False, copy="True") ###create tilt file #get image size from the first image imagesizex = tiltdata[0]['image'].shape[1] imagesizey = tiltdata[0]['image'].shape[0] #shift half tilt series relative to eachother #SS I'm arbitrarily making the bin parameter here 1 because it's not necessary to sample at this point shifts = apTomo.getGlobalShift(ordered_imagelist, 1, refimg) #OPTION: refinement might be more robust by doing one round of IMOD aligment to prealign images before doing protomo refine origins = convertShiftsToOrigin(shifts, imagesizex, imagesizey) writeTiltFile2(tiltfilename, seriesname, newfilenames, origins, tilts, azimuth, refimg) return tilts, accumulated_dose_list, new_ordered_imagelist, maxtilt
def doseCompensate(seriesname, rundir, sessionname, tiltseriesnumber, frame_aligned_images, raw_path, pixelsize, dose_presets, dose_a, dose_b, dose_c): """ Images will be lowpass filtered using equation (3) from Grant & Grigorieff, 2015. No changes to the database are made. No backups are made. """ sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata) tiltdata = apTomo.getImageList([tiltseriesdata]) frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata) tilts, ordered_imagelist, accumulated_dose_list, ordered_mrc_files, refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned="False") if frame_aligned_images == "True": #For different image filenames a, ordered_imagelist, c, d, e = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images) newfilenames, new_ordered_imagelist = apProTomo.getImageFiles(ordered_imagelist, raw_path, link=False, copy=False) if (dose_presets == "Light"): dose_a = 0.245 dose_b = -1.6 dose_c = 12 elif (dose_presets == "Moderate"): dose_a = 0.245 dose_b = -1.665 dose_c = 2.81 elif (dose_presets == "Heavy"): dose_a = 0.245 dose_b = -1.4 dose_c = 2 apDisplay.printMsg('Dose compensating all tilt images with a=%s, b=%s, and c=%s...' % (dose_a, dose_b, dose_c)) for image, j in zip(new_ordered_imagelist, range(len(new_ordered_imagelist))): lowpass = float(np.real(complex(dose_a/(accumulated_dose_list[j] - dose_c))**(1/dose_b))) #equation (3) from Grant & Grigorieff, 2015 if lowpass < 0.0: lowpass = 0.0 im = mrc.read(image) im = imagefilter.lowPassFilter(im, apix=pixelsize, radius=lowpass, msg=False) im=imagenorm.normStdev(im) mrc.write(im, image) #Make plots apProTomo2Aligner.makeDosePlots(rundir, seriesname, tilts, accumulated_dose_list, dose_a, dose_b, dose_c) apDisplay.printMsg("Dose compensation finished for tilt-series #%s!" % tiltseriesnumber) return
def commitToJensenDatabase(session_time, fulltomodata, stackdir, processdir, stackname, description): if not upload_to_Jensen_database: return raptordatabase = 0 apDisplay.printMsg("Uploding to Jensen tomography database") alignrun = fulltomodata['alignrun'] tiltseries = fulltomodata['tiltseries'] nn = len(stackname) mrcbase = processdir + '/align/' + stackname[0:-3] mrcpath = mrcbase + '_full.rec' if not os.path.exists(mrcpath): mrcpath = mrcbase + '_part.rec' if not os.path.exists(mrcpath): mrcpath = '' if mrcpath == '': apDisplay.printWarning('No output full or part mrc found.') raptordatabase = 1 else: print '!!!0', ' mrcpath=', mrcpath print '!!!1', ' stackdir=', stackdir, ' stackname=', stackname, ' processdir=', processdir, ' description=', description imagelist = apTomo.getImageList([tiltseries]) defocus = imagelist[0]['preset']['defocus'] magnification = imagelist[0]['preset']['magnification'] q = appiondata.ApTiltsInAlignRunData(alignrun=alignrun) r = q.query() tomosettings = r[0]['settings'] tilt_min = tomosettings['tilt min'] tilt_max = tomosettings['tilt max'] tilt_step = tomosettings['tilt step'] dose = tomosettings['dose'] print '!!!2', ' defocus=', defocus, ' magnification=', magnification, ' tilt_min=', tilt_min, ' tilt_max=', tilt_max, ' tilt_step=', tilt_step, ' dose=', dose #sessiondata = tiltdatalist[0]['session'] session_id = tiltseries.dbid print '!!!3', ' session_id=', session_id, ' session_time=', session_time return raptordatabase
def start(self): ###do queries sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata) tiltseriessettings= apTomo.getTomographySettings(sessiondata,tiltseriesdata) tiltdata=apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") print "ordering list" tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata) #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int ###set up files seriesname='series'+str(self.params['tiltseries']) tiltfilename=seriesname+'.tlt' param_out=seriesname+'.param' maxtilt=max([abs(tilts[0]),abs(tilts[-1])]) apDisplay.printMsg("highest tilt angle is %f" % maxtilt) self.params['cos_alpha']=math.cos(maxtilt*math.pi/180) self.params['raw_path']=os.path.join(self.params['rundir'],'raw') rawexists=apParam.createDirectory(self.params['raw_path']) apDisplay.printMsg("copying raw images") newfilenames=apProTomo.getImageFiles(ordered_imagelist,self.params['raw_path'], link=False) #get alignment data alignerdata = apTomo.getAlignerdata(self.params['alignerid']) imgshape = apTomo.getTomoImageShape(ordered_imagelist[0]) imgcenter = {'x':self.imgshape[1]/2,'y':self.imgshape[0]/2} specimen_euler, azimuth, origins, rotations = apTomo.getAlignmentFromDB(alignerdata,imgcenter) #write protomo2 tilt file outtltfile='series.tlt' seriesname='series' apProTomo.writeTiltFile2(outfilename, seriesname, specimen_eulers, azimuth, referenceimage )
def commitToJensenDatabase (session_time, fulltomodata, stackdir, processdir, stackname, description ): if not upload_to_Jensen_database: return raptordatabase = 0 apDisplay.printMsg("Uploding to Jensen tomography database") alignrun = fulltomodata['alignrun'] tiltseries = fulltomodata['tiltseries'] nn = len(stackname) mrcbase = processdir + '/align/' + stackname[0:-3] mrcpath = mrcbase + '_full.rec' if not os.path.exists(mrcpath): mrcpath = mrcbase + '_part.rec' if not os.path.exists(mrcpath): mrcpath = '' if mrcpath == '': apDisplay.printWarning('No output full or part mrc found.') raptordatabase = 1 else: print '!!!0', ' mrcpath=', mrcpath print '!!!1', ' stackdir=', stackdir, ' stackname=', stackname, ' processdir=', processdir, ' description=',description imagelist = apTomo.getImageList([tiltseries]) defocus = imagelist[0]['preset']['defocus'] magnification = imagelist[0]['preset']['magnification'] q = appiondata.ApTiltsInAlignRunData(alignrun=alignrun) r = q.query() tomosettings = r[0]['settings'] tilt_min = tomosettings['tilt min'] tilt_max = tomosettings['tilt max'] tilt_step = tomosettings['tilt step'] dose = tomosettings['dose'] print '!!!2', ' defocus=', defocus, ' magnification=', magnification, ' tilt_min=', tilt_min, ' tilt_max=', tilt_max, ' tilt_step=', tilt_step, ' dose=', dose #sessiondata = tiltdatalist[0]['session'] session_id = tiltseries.dbid print '!!!3', ' session_id=', session_id, ' session_time=', session_time return raptordatabase
def start(self): commit = self.params['commit'] tiltdatalist = apTomo.getTiltdataList(self.params['tiltseries'],self.params['othertiltseries']) sessiondata = tiltdatalist[0]['session'] description = self.params['description'] alignsample = self.params['sample'] cycle = self.params['cycle'] alignmethod = self.params['alignmethod'] apDisplay.printMsg("getting imagelist") imagelist = apTomo.getImageList(tiltdatalist) tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(imagelist) # This parameter is needed for protomo, but not protomo2 if self.params['refimg']: refimg = self.params['refimg'] if alignmethod != 'protomo2': for file in ordered_mrc_files: # protomo can not function with a negative origin # protomo2 CAN function with negative origin apImage.shiftMRCStartToZero(file) apDisplay.printMsg("getting pixelsize") pixelsize = apTomo.getTomoPixelSize(ordered_imagelist[refimg]) if pixelsize is None: apDisplay.printError('Pixel Size not retrieved. Invalid tilt series for processing') imgshape = apTomo.getTomoImageShape(ordered_imagelist[refimg]) corr_bin = apTomo.getCorrelatorBinning(imgshape) center = {'x':imgshape[1]/2,'y':imgshape[0]/2} default_azimuth = apTomo.getDefaultAzimuthFromLeginon(ordered_imagelist[refimg]) processdir = os.path.abspath(self.params['rundir']) imodseriesname = apTomo.getFilename(tiltdatalist) # protomo2 does not write out text files, intermediate info is in memory and the final result is binary seriesname = 'tomo'+ imodseriesname # Write tilt series stack images and tilt angles, not needed for protomo2 if alignmethod != 'protomo2': stackdir = self.params['tiltseriesdir'] stackname = imodseriesname+".st" apTomo.writeTiltSeriesStack(stackdir,stackname,ordered_mrc_files,1e10*pixelsize) apImod.writeRawtltFile(stackdir,imodseriesname,tilts) leginonxcorrlist = [] for tiltdata in tiltdatalist: settingsdata = apTomo.getTomographySettings(sessiondata,tiltdata) leginonxcorrlist.append(settingsdata) # Run protomo2 if alignmethod == 'protomo2': self.runProtomo2(sessiondata, processdir, seriesname, ordered_imagelist, refimg, center, corr_bin, commit, tilts) # protomo2 does not need anything beyond this point, so exit return if cycle != 1 or alignmethod != 'protomo': cycles=[cycle,] else: # also process and commit protomo cycle 0 if doing cycle 1 cycles=[0,1] for cycle in cycles: if alignmethod == 'protomo' or alignmethod == 'leginon': self.params['aligndir'],self.params['imagedir'] = apProTomo.setProtomoDir(self.params['rundir'],cycle) aligndir = self.params['aligndir'] # Link images into rundir/raw rawimagenames = apProTomo.linkImageFiles(ordered_imagelist,self.params['imagedir']) tltfile = os.path.join(aligndir,seriesname+'-%02d-itr.tlt' % (cycle,)) if cycle == 0: # get initial shift alignment from leginon tiltcorrelator # Assume all tiltdata have the same tomography settings shifts = apTomo.getGlobalShift(ordered_imagelist, corr_bin, refimg) tltparams = apProTomo.convertShiftsToParams(tilts,shifts,center,default_azimuth,rawimagenames) apProTomo.writeTiltFile(tltfile,seriesname, tltparams[0], tltparams[1]) refineparamdict=apProTomo.createRefineDefaults(len(tilts), os.path.join(processdir,'raw'),os.path.join(processdir,'out')) refineparamdict = apProTomo.updateRefineParams(refineparamdict,imgshape,alignsample,100,refimg) else: lasttltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (cycle-1,)) if self.params['goodcycle']: if (self.params['goodstart'] == 0 and self.params['goodend'] == len(ordered_imagelist) - 1): # revert goodcycle related params since nothing will be reset self.params['goodcycle'] = None self.params['goodstart'] = None self.params['goodend'] = None if not self.params['goodcycle']: #default uses last cycle shutil.copy(lasttltfile,tltfile) else: if not (self.params['goodstart'] or self.params['goodend']): shutil.copy(lasttltfile,tltfile) else: # Reset bad ending tilts before alignment cycle goodtltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (self.params['goodcycle'],)) goodtltparams = apProTomo.parseTilt(goodtltfile) lasttltparams = apProTomo.parseTilt(lasttltfile) tltparams = apProTomo.resetTiltParams(lasttltparams,goodtltparams,self.params['goodstart'],self.params['goodend']) apProTomo.writeTiltFile(tltfile,seriesname, tltparams[0], tltparams[1]) lastrefineparamfile = os.path.join(aligndir,seriesname+'-%02d.param' % (cycle-1,)) refineparamdict = apProTomo.parseRefineParamFile(lastrefineparamfile) refineparamdict = apProTomo.updateRefineParams(refineparamdict,imgshape,alignsample,self.params['region'],refimg) # Write param file in rundir/align paramfilepath = os.path.join(seriesname+'-%02d.param' % (cycle)) fullparamfilepath = os.path.join(aligndir,paramfilepath) apProTomo.writeRefineParamFile(refineparamdict,fullparamfilepath) # run porjalign script os.chdir(aligndir) if cycle > 0: self.runProjalign(paramfilepath) else: lasttltfile = os.path.join(aligndir,seriesname+'-%02d-itr.tlt' % (cycle,)) newtltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (cycle,)) shutil.copy(lasttltfile,newtltfile) # convert to imod alignment alignpathprefix = os.path.join(processdir, seriesname) centertuple = (center['x'],center['y']) apProTomo.convertGlobalTransformProtomoToImod(seriesname+'-%02d' %(cycle),imodseriesname, centertuple) elif alignmethod == 'imod-shift': # Correlation by Coarse correlation in IMOD aligndir = processdir imodxcorrdata = apImod.coarseAlignment(stackdir, processdir, imodseriesname, commit) # Global Transformation gtransforms = apImod.convertToGlobalAlignment(processdir, imodseriesname) # Create Aligned Stack for record bin = int(math.ceil(min(imgshape) / 512.0)) apImod.createAlignedStack(stackdir, aligndir, imodseriesname,bin) if alignmethod == 'protomo' or alignmethod == 'leginon': alifilename = imodseriesname+'-%02d.ali' % cycle os.rename(imodseriesname+'.ali',alifilename) else: alifilename = imodseriesname+'.ali' alifilepath = os.path.join(aligndir,alifilename) # commit to database if commit: if alignmethod == 'protomo' or alignmethod == 'leginon': # -- Commit Parameters -- protomodata = apProTomo.insertProtomoParams(seriesname) alignrun = apTomo.insertTomoAlignmentRun(sessiondata,leginonxcorrlist[0],None,protomodata,None,1,self.params['runname'],self.params['rundir'],self.params['description']) self.cycle_description = self.params['description'] self.params['cycle'] = cycle if cycle == 0: # temporarily change aligner description on the initial 0 cycle self.params['description'] = 'leginon correlation results' # insert sample and window size and all the other user defined params into ApProtomoRefinementParamsData alignerdata = apProTomo.insertAlignIteration(alignrun, protomodata, self.params, refineparamdict,ordered_imagelist[refimg]) # -- Commit Results -- resulttltfile = os.path.join(aligndir,seriesname+'-%02d-fitted.tlt' % (cycle,)) resulttltparams = apProTomo.parseTilt(resulttltfile) if resulttltparams: # commit the geometry parameters (psi, theta, phi, azimuth) modeldata = apProTomo.insertModel(alignerdata, resulttltparams) # insert results into ApProtomoAlignmentData (also used by imod) for each image for i,imagedata in enumerate(ordered_imagelist): apProTomo.insertTiltAlignment(alignerdata,imagedata,i,resulttltparams[0][i],center) self.params['description'] = self.cycle_description else: alignrun = apTomo.insertTomoAlignmentRun(sessiondata,None,imodxcorrdata,None,None,1,self.params['runname'],self.params['rundir'],self.params['description']) alignerdata = apTomo.insertAlignerParams(alignrun,self.params) #results prexgfile = os.path.join(aligndir,imodseriesname+'.prexg') shifts = apImod.readShiftPrexgFile(aligndir, imodseriesname) resulttltparams = apProTomo.convertShiftsToParams(tilts,shifts,center,default_azimuth) if resulttltparams: modeldata = apProTomo.insertModel(alignerdata, resulttltparams) for i,imagedata in enumerate(ordered_imagelist): apProTomo.insertTiltAlignment(alignerdata,imagedata,i,resulttltparams[0][i],center) # multiple tilt series in one alignrun for i in range(0,len(tiltdatalist)): if i == 0: primary = True else: primary = False apTomo.insertTiltsInAlignRun(alignrun, tiltdatalist[i],leginonxcorrlist[i],primary) apTomo.makeAlignStackMovie(alifilepath) os.chdir(processdir)
def start(self): # set local parameters commit = self.params['commit'] tiltdatalist = apTomo.getTiltdataList(self.params['tiltseries'], self.params['othertiltseries']) sessiondata = tiltdatalist[0]['session'] description = self.params['description'] runname = self.params['runname'] alignmethod = self.params['alignmethod'] reconbin = int(self.params['reconbin']) thickness_pixel = int(self.params['reconthickness']) markersize_nm = int(self.params['markersize']) markernumber = int(self.params['markernumber']) apDisplay.printMsg("getting imagelist") imagelist = apTomo.getImageList(tiltdatalist) tilts, ordered_imagelist, ordered_mrc_files, refimg = apTomo.orderImageList( imagelist) apDisplay.printMsg("getting pixelsize") pixelsize = apTomo.getTomoPixelSize(ordered_imagelist[refimg]) imgshape = apTomo.getTomoImageShape(ordered_imagelist[refimg]) #thickness_binnedpixel = int(thickness_nm * 1e-9 / (pixelsize * reconbin)) markersize_pixel = int(markersize_nm * 1e-9 / pixelsize) processdir = os.path.abspath(self.params['rundir']) imodseriesname = apTomo.getFilename(tiltdatalist) seriesname = imodseriesname # Write tilt series stack images and tilt angles stackdir = self.params['tiltseriesdir'] stackname = imodseriesname + ".st" apTomo.writeTiltSeriesStack(stackdir, stackname, ordered_mrc_files, 1e10 * pixelsize) apRaptor.linkStToMrcExtension(stackdir, imodseriesname) apImod.writeRawtltFile(stackdir, imodseriesname, tilts) # Get Leginon tomography settings leginontomosettingslist = [] for tiltdata in tiltdatalist: settingsdata = apTomo.getTomographySettings(sessiondata, tiltdata) leginontomosettingslist.append(settingsdata) aligndir = processdir # run the script and get alignment results when raptor can output alignment results in the future. raptoraligndata is None for now. returncode, raptoraligndata, raptorfailed = apRaptor.alignAndRecon( stackdir, stackname, processdir, markersize_pixel, reconbin, thickness_pixel, markernumber, commit) # Create Aligned Stack for record, not done in apRaptor yet, currently raptoraligndata is None if not raptorfailed: alifilename = imodseriesname + '.ali' alifilepath = os.path.join(aligndir, 'align', alifilename) print alifilepath # commit to database if commit: # parameters raptorparamsdata = apRaptor.insertRaptorParams( markersize_nm, markernumber) alignrun = apTomo.insertTomoAlignmentRun( sessiondata, None, None, None, raptorparamsdata, 1, self.params['runname'], self.params['rundir'], self.params['description'], raptorfailed) # to accomodate iterative alignment, one alignmentrun may have # used the aligner several times, for this case a single # aligner params data is inserted as in the case of Imod xcorr alignerdata = apTomo.insertAlignerParams(alignrun, self.params) #results if raptoraligndata: # if raptor has alignment result, it is converted to protomo # format which is more parameterized and saved prexgfile = os.path.join(aligndir, imodseriesname + '.prexg') shifts = apImod.readShiftPrexgFile(aligndir, imodseriesname) resulttltparams = apProTomo.convertShiftsToParams( tilts, shifts, center) if resulttltparams: modeldata = apProTomo.insertModel(alignerdata, resulttltparams) for i, imagedata in enumerate(ordered_imagelist): apProTomo.insertTiltAlignment(alignerdata, imagedata, i, resulttltparams[0][i], center) # multiple tilt series in one alignrun for i in range(0, len(tiltdatalist)): if i == 0: primary = True else: primary = False # Record tilts in align run allows more than one tilt series to be # used in one align run. apTomo.insertTiltsInAlignRun(alignrun, tiltdatalist[i], leginontomosettingslist[i], primary) if not raptorfailed: apTomo.makeAlignStackMovie(alifilepath) os.chdir(processdir) # Full tomogram created with raptor is ???? handness????? if not raptorfailed: ''' voltransform = '????' origtomopath = os.path.join(processdir, seriesname+"_full.rec") currenttomopath = apImod.transformVolume(origtomopath,voltransform) shutil.move(currenttomopath, origtomopath) ''' zprojectfile = apImod.projectFullZ(processdir, runname, seriesname, reconbin, False, False) try: zimagedata = apTomo.uploadZProjection( runname, imagelist[0], zprojectfile) except: zimagedata = None fullrundata = apTomo.insertFullTomoRun(sessiondata, processdir, runname, 'imod-wbp') fulltomodata = apTomo.insertFullTomogram( sessiondata, tiltdatalist[0], alignerdata, fullrundata, runname, description, zimagedata, thickness_pixel, reconbin) # if raptor succeeded, upload data and parameters to database session_time = sessiondata.timestamp description = self.params['description'] raptordatabase = apRaptor.commitToJensenDatabase( session_time, fulltomodata, stackdir, processdir, stackname, description) if raptordatabase == 0: apDisplay.printMsg( "RAPTOR and uploading to Jensen database done.") else: apDisplay.printWarning( "Uploading to Jensen database failed.")
def start(self): # set local parameters commit = self.params['commit'] tiltdatalist = apTomo.getTiltdataList(self.params['tiltseries'],self.params['othertiltseries']) sessiondata = tiltdatalist[0]['session'] description = self.params['description'] runname = self.params['runname'] alignmethod = self.params['alignmethod'] reconbin = int(self.params['reconbin']) thickness_pixel = int(self.params['reconthickness']) markersize_nm = int(self.params['markersize']) markernumber = int(self.params['markernumber']) apDisplay.printMsg("getting imagelist") imagelist = apTomo.getImageList(tiltdatalist) tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(imagelist) apDisplay.printMsg("getting pixelsize") pixelsize = apTomo.getTomoPixelSize(ordered_imagelist[refimg]) imgshape = apTomo.getTomoImageShape(ordered_imagelist[refimg]) #thickness_binnedpixel = int(thickness_nm * 1e-9 / (pixelsize * reconbin)) markersize_pixel = int(markersize_nm * 1e-9 / pixelsize) processdir = os.path.abspath(self.params['rundir']) imodseriesname = apTomo.getFilename(tiltdatalist) seriesname = imodseriesname # Write tilt series stack images and tilt angles stackdir = self.params['tiltseriesdir'] stackname = imodseriesname+".st" apTomo.writeTiltSeriesStack(stackdir,stackname,ordered_mrc_files,1e10*pixelsize) apRaptor.linkStToMrcExtension(stackdir,imodseriesname) apImod.writeRawtltFile(stackdir,imodseriesname,tilts) # Get Leginon tomography settings leginontomosettingslist = [] for tiltdata in tiltdatalist: settingsdata = apTomo.getTomographySettings(sessiondata,tiltdata) leginontomosettingslist.append(settingsdata) aligndir = processdir # run the script and get alignment results when raptor can output alignment results in the future. raptoraligndata is None for now. returncode, raptoraligndata, raptorfailed = apRaptor.alignAndRecon(stackdir, stackname, processdir, markersize_pixel, reconbin, thickness_pixel, markernumber, commit) # Create Aligned Stack for record, not done in apRaptor yet, currently raptoraligndata is None if not raptorfailed: alifilename = imodseriesname+'.ali' alifilepath = os.path.join(aligndir,'align',alifilename) print alifilepath # commit to database if commit: # parameters raptorparamsdata = apRaptor.insertRaptorParams(markersize_nm,markernumber) alignrun = apTomo.insertTomoAlignmentRun(sessiondata,None,None,None,raptorparamsdata,1,self.params['runname'],self.params['rundir'],self.params['description'],raptorfailed) # to accomodate iterative alignment, one alignmentrun may have # used the aligner several times, for this case a single # aligner params data is inserted as in the case of Imod xcorr alignerdata = apTomo.insertAlignerParams(alignrun,self.params) #results if raptoraligndata: # if raptor has alignment result, it is converted to protomo # format which is more parameterized and saved prexgfile = os.path.join(aligndir,imodseriesname+'.prexg') shifts = apImod.readShiftPrexgFile(aligndir, imodseriesname) resulttltparams = apProTomo.convertShiftsToParams(tilts,shifts,center) if resulttltparams: modeldata = apProTomo.insertModel(alignerdata, resulttltparams) for i,imagedata in enumerate(ordered_imagelist): apProTomo.insertTiltAlignment(alignerdata,imagedata,i,resulttltparams[0][i],center) # multiple tilt series in one alignrun for i in range(0,len(tiltdatalist)): if i == 0: primary = True else: primary = False # Record tilts in align run allows more than one tilt series to be # used in one align run. apTomo.insertTiltsInAlignRun(alignrun, tiltdatalist[i],leginontomosettingslist[i],primary) if not raptorfailed: apTomo.makeAlignStackMovie(alifilepath) os.chdir(processdir) # Full tomogram created with raptor is ???? handness????? if not raptorfailed: ''' voltransform = '????' origtomopath = os.path.join(processdir, seriesname+"_full.rec") currenttomopath = apImod.transformVolume(origtomopath,voltransform) shutil.move(currenttomopath, origtomopath) ''' zprojectfile = apImod.projectFullZ(processdir, runname, seriesname,reconbin,False,False) try: zimagedata = apTomo.uploadZProjection(runname,imagelist[0],zprojectfile) except: zimagedata = None fullrundata = apTomo.insertFullTomoRun(sessiondata,processdir,runname,'imod-wbp') fulltomodata = apTomo.insertFullTomogram(sessiondata,tiltdatalist[0],alignerdata, fullrundata,runname,description,zimagedata,thickness_pixel,reconbin) # if raptor succeeded, upload data and parameters to database session_time = sessiondata.timestamp description = self.params['description'] raptordatabase = apRaptor.commitToJensenDatabase(session_time, fulltomodata, stackdir, processdir, stackname, description) if raptordatabase == 0: apDisplay.printMsg("RAPTOR and uploading to Jensen database done.") else: apDisplay.printWarning("Uploading to Jensen database failed.")
def getTiltSeriesFromId(tiltid): seriesdata=leginon.leginondata.TiltSeriesData.direct_query(tiltid) imgtree=apTomo.getImageList([seriesdata]) return imgtree
def ctfCorrect(seriesname, rundir, projectid, sessionname, tiltseriesnumber, tiltfilename, frame_aligned_images, pixelsize, DefocusTol, iWidth, amp_contrast): """ Leginondb will be queried to get the 'best' defocus estimate on a per-image basis. Confident defoci will be gathered and unconfident defoci will be interpolated. Images will be CTF corrected by phase flipping using ctfphaseflip from the IMOD package. A plot of the defocus values will is made. A CTF plot using the mean defocus is made. """ try: apDisplay.printMsg('CTF correcting all tilt images using defocus values from Leginon database...') os.chdir(rundir) raw_path=rundir+'/raw/' ctfdir='%s/ctf_correction/' % rundir os.system("mkdir %s" % ctfdir) defocus_file_full=ctfdir+seriesname+'_defocus.txt' tilt_file_full=ctfdir+seriesname+'_tilts.txt' image_list_full=ctfdir+seriesname+'_images.txt' uncorrected_stack=ctfdir+'stack_uncorrected.mrc' corrected_stack=ctfdir+'stack_corrected.mrc' out_full=ctfdir+'out' log_file_full=ctfdir+'ctf_correction.log' project='ap'+projectid sinedon.setConfig('appiondata', db=project) sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata) tiltdata = apTomo.getImageList([tiltseriesdata]) frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata) tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images) cs = tiltdata[0]['scope']['tem']['cs']*1000 voltage = int(tiltdata[0]['scope']['high tension']/1000) if os.path.isfile(ctfdir+'out/out01.mrc'): #Throw exception if already ctf corrected sys.exit() #Get tilt azimuth cmd="awk '/TILT AZIMUTH/{print $3}' %s" % (tiltfilename) proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) (tilt_azimuth, err) = proc.communicate() tilt_azimuth=float(tilt_azimuth) estimated_defocus=[] for image in range(len(ordered_imagelist)): imgctfdata=ctfdb.getBestCtfValue(ordered_imagelist[image], msg=False) try: if imgctfdata['resolution_50_percent'] < 100.0: #if there's a yellow ring in Appion, trust defocus estimation estimated_defocus.append((imgctfdata['defocus1']+imgctfdata['defocus2'])*1000000000/2) else: #Poorly estimated. Guess its value later estimated_defocus.append(999999999) except: #No data. Guess its value later estimated_defocus.append(999999999) #Find mean and stdev to prune out confident defocus values that are way off defocus_stats_list=filter(lambda a: a != 999999999, estimated_defocus) avg=np.array(defocus_stats_list).mean() stdev=np.array(defocus_stats_list).std() good_tilts=[] good_defocus_list=[] for tilt, defocus in zip(tilts, estimated_defocus): if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev): good_defocus_list.append(defocus) good_tilts.append(tilt) #Using a linear best fit because quadratic and cubic go off the rails. Estimation doesn't need to be extremely accurate anyways. x=np.linspace(int(round(tilts[0])), int(round(tilts[len(tilts)-1])), 1000) s=scipy.interpolate.UnivariateSpline(good_tilts,good_defocus_list,k=1) y=s(x) #Make defocus list with good values and interpolations for bad values finished_defocus_list=[] for tilt, defocus in zip(tilts, estimated_defocus): if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev): finished_defocus_list.append(int(round(defocus))) else: #Interpolate finished_defocus_list.append(int(round(y[int(round(tilt))]))) new_avg=np.array(finished_defocus_list).mean() new_stdev=np.array(finished_defocus_list).std() #Write defocus file, tilt file, and image list file for ctfphaseflip and newstack f = open(defocus_file_full,'w') f.write("%d\t%d\t%.2f\t%.2f\t%d\t2\n" % (1,1,tilts[0],tilts[0],finished_defocus_list[0])) for i in range(1,len(tilts)): f.write("%d\t%d\t%.2f\t%.2f\t%d\n" % (i+1,i+1,tilts[i],tilts[i],finished_defocus_list[i])) f.close() f = open(tilt_file_full,'w') for tilt in tilts: f.write("%.2f\n" % tilt) f.close() mrc_list=[] presetname=tiltdata[0]['preset']['name'] for image in ordered_mrc_files: mrcname=presetname+image.split(presetname)[-1] mrc_list.append(raw_path+'/'+mrcname) f = open(image_list_full,'w') f.write("%d\n" % len(tilts)) for filename in mrc_list: f.write(filename+'\n') f.write("%d\n" % 0) f.close() #Rotate and pad images so that they are treated properly by ctfphaseflip. apDisplay.printMsg("Preparing images for IMOD...") for filename in mrc_list: image=mrc.read(filename) dimx=len(image[0]) dimy=len(image) #First rotate 90 degrees in counter-clockwise direction. This makes it so positive angle images are higher defocused on the right side of the image image=np.rot90(image, k=-1) #Rotate image and write image=imrotate(image, -tilt_azimuth, order=1) #Linear interpolation is fastest and there is barely a difference between linear and cubic mrc.write(image, filename) f = open(log_file_full,'w') #Make stack for correction,phase flip, extract images, replace images cmd1="newstack -fileinlist %s -output %s > %s" % (image_list_full, uncorrected_stack, log_file_full) f.write("%s\n\n" % cmd1) print cmd1 subprocess.check_call([cmd1], shell=True) cmd2="ctfphaseflip -input %s -output %s -AngleFile %s -defFn %s -pixelSize %s -volt %s -DefocusTol %s -iWidth %s -SphericalAberration %s -AmplitudeContrast %s 2>&1 | tee %s" % (uncorrected_stack, corrected_stack, tilt_file_full, defocus_file_full, pixelsize/10, voltage, DefocusTol, iWidth, cs, amp_contrast, log_file_full) f.write("\n\n%s\n\n" % cmd2) print cmd2 subprocess.check_call([cmd2], shell=True) cmd3="newstack -split 1 -append mrc %s %s >> %s" % (corrected_stack, out_full, log_file_full) f.write("\n\n%s\n\n" % cmd3) print cmd3 subprocess.check_call([cmd3], shell=True) f.write("\n\n") apDisplay.printMsg("Overwriting uncorrected raw images with CTF corrected images") new_images=glob.glob(ctfdir+'out*mrc') new_images.sort() #Unrotate and unpad images for filename in new_images: image=mrc.read(filename) image=imrotate(image, tilt_azimuth, order=1) image=np.rot90(image, k=1) big_dimx=len(image[0]) big_dimy=len(image) cropx1=int((big_dimx-dimx)/2) cropx2=int(dimx+(big_dimx-dimx)/2) cropy1=int((big_dimy-dimy)/2) cropy2=int(dimy+(big_dimy-dimy)/2) image=image[cropy1:cropy2,cropx1:cropx2] mrc.write(image, filename) for i in range(len(new_images)): cmd4="rm %s; ln %s %s" % (mrc_list[i], new_images[i], mrc_list[i]) f.write("%s\n" % cmd4) os.system(cmd4) #Make plots apProTomo2Aligner.makeDefocusPlot(rundir, seriesname, defocus_file_full) apProTomo2Aligner.makeCTFPlot(rundir, seriesname, defocus_file_full, voltage, cs) cleanup="rm %s %s" % (uncorrected_stack, corrected_stack) os.system(cleanup) output1="%.2f%% of the images for tilt-series #%s had poor defocus estimates or fell outside of one standard deviation from the original mean." % (100*(len(estimated_defocus)-len(defocus_stats_list))/len(estimated_defocus), tiltseriesnumber) output2="The defocus mean and standard deviation for tilt-series #%s after interpolating poor values is %.2f and %.2f microns, respectively." % (tiltseriesnumber, new_avg/1000, new_stdev/1000) f.write("\n");f.write(output1);f.write("\n");f.write(output2);f.write("\n");f.close() apDisplay.printMsg(output1) apDisplay.printMsg(output2) apDisplay.printMsg("CTF correction finished for tilt-series #%s!" % tiltseriesnumber) except subprocess.CalledProcessError: apDisplay.printError("An IMOD command failed, so CTF correction could not be completed. Make sure IMOD is in your $PATH.") except SystemExit: apDisplay.printWarning("It looks like you've already CTF corrected tilt-series #%s. Skipping CTF correction!" % tiltseriesnumber) except: apDisplay.printError("CTF correction could not be completed. Make sure IMOD, numpy, and scipy are in your $PATH. Make sure defocus has been estimated through Appion.\n")
def start(self): ### some of this should go in preloop functions ###do queries sessiondata = apDatabase.getSessionDataFromSessionName( self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId( self.params['tiltseries'], sessiondata) tiltseriessettings = apTomo.getTomographySettings( sessiondata, tiltseriesdata) tiltdata = apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") print "ordering list" tilts, ordered_imagelist, ordered_mrc_files, refimg = apTomo.orderImageList( tiltdata) #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int ###set up files seriesname = self.params['seriesname'] print type(seriesname) # param_out=seriesname+'.param' ###insert protomo run params print "first insert" protomodata = apProTomo.insertProtomoParams(seriesname) print "second insert" alignrun = apTomo.insertTomoAlignmentRun(sessiondata, tiltseriessettings, None, protomodata, None, 1, self.params['runname'], self.params['rundir'], self.params['description']) ###insert protomo alignment ###hack to get around need to parse protomo param file #should read imgref from tlt file refineparamdict = { 'alismp': None, 'alibox_x': None, 'alibox_y': None, 'cormod': None, 'imgref': None } ### self.params['goodcycle'] = None if self.params['goodrange'] is None: self.params['goodstart'] = 1 self.params['goodend'] = len(tilts) alignerdata = apProTomo.insertAlignIteration(alignrun, protomodata, self.params, refineparamdict, ordered_imagelist[refimg]) # read tlt file print "third insert" alignmentdict, geometrydict, seriesname = apProTomo.parseTilt( self.params['tltfile']) # insert geometry model modeldata = apProTomo.insertModel2(alignerdata, geometrydict) #insert image alignments for i, imagedata in enumerate(ordered_imagelist): #Caution...assumes ordered_imagelist is in same order as tlt file apProTomo.insertTiltAlignment(alignerdata, imagedata, i, alignmentdict[i + 1], center=None) print "fourth insert" apTomo.insertTiltsInAlignRun(alignrun, tiltseriesdata, tiltseriessettings, True)
def getTiltSeriesFromId(tiltid): seriesdata = leginon.leginondata.TiltSeriesData.direct_query(tiltid) imgtree = apTomo.getImageList([seriesdata]) return imgtree