def checkConflicts(self): sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.sessiondata = sessiondata if self.params['goodalignerid'] is None: if self.params['tiltseriesnumber'] is None : apDisplay.printError("There is no tilt series specified") if self.params['runname'] is None: apDisplay.printError("enter a run name") if self.params['description'] is None: apDisplay.printError("enter a description, e.g. --description='awesome data'") tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseriesnumber'],sessiondata) self.params['tiltseries'] = tiltdata if self.params['othertilt'] is not None: othertiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['othertilt'],sessiondata) self.params['othertiltseries'] = othertiltdata else: self.params['othertiltseries'] = None if self.params['alignmethod'] == 'leginon': self.params['cycle'] = 0 self.params['region'] = 100 else: self.createParamsFromGoodAligner(self.params['goodalignerid']) if self.params['goodcycle'] is None: if self.params['goodstart'] or self.params['goodend']: apDisplay.printError("Cannot specify reseting image range without knowing the cycle number to reset to") elif self.params['goodcycle'] >= self.params['cycle']: apDisplay.printError("Only older cycle can be used as the base to reset image origin to") # Sampling is needed for protomo1 and 2, not imod if self.params['sample'] is not None: if self.params['sample'] < 1: apDisplay.printError("Sampling factor (--sample) must be greater or equal to 1.")
def checkConflicts(self): sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.sessiondata = sessiondata if self.params['tiltseriesnumber'] is None : apDisplay.printError("There is no tilt series specified") if self.params['runname'] is None: apDisplay.printError("enter a run name") if self.params['description'] is None: apDisplay.printError("enter a description, e.g. --description='awesome data'") tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseriesnumber'],sessiondata) self.params['tiltseries'] = tiltdata if self.params['othertilt'] is not None: othertiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['othertilt'],sessiondata) self.params['othertiltseries'] = othertiltdata else: self.params['othertiltseries'] = None
def checkConflicts(self): sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.sessiondata = sessiondata if self.params['tiltseriesnumber'] is None : apDisplay.printError("There is no tilt series specified") if self.params['runname'] is None: apDisplay.printError("enter a run name") if self.params['description'] is None: apDisplay.printError("enter a description, e.g. --description='awesome data'") tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseriesnumber'],sessiondata) self.params['tiltseries'] = tiltdata if self.params['othertilt'] is not None: othertiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['othertilt'],sessiondata) self.params['othertiltseries'] = othertiltdata else: self.params['othertiltseries'] = None
def getImageShapeFromTiltSeries(self): sessiondata = apDatabase.getSessionDataFromSessionName( self.params['sessionname']) tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId( self.params['tiltseriesnumber'], sessiondata) firstimagedata = apTomo.getFirstImage(tiltdata) self.imageshape = apTomo.getTomoImageShape(firstimagedata)
def start(self): ### some of this should go in preloop functions ###do queries sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata) tiltseriessettings= apTomo.getTomographySettings(sessiondata,tiltseriesdata) tiltdata=apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") print "ordering list" tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata) #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int ###set up files seriesname=self.params['seriesname'] print type(seriesname) # param_out=seriesname+'.param' ###insert protomo run params print "first insert" protomodata = apProTomo.insertProtomoParams(seriesname) print "second insert" alignrun = apTomo.insertTomoAlignmentRun(sessiondata,tiltseriessettings,None,protomodata,None,1,self.params['runname'],self.params['rundir'],self.params['description']) ###insert protomo alignment ###hack to get around need to parse protomo param file #should read imgref from tlt file refineparamdict={'alismp':None,'alibox_x':None,'alibox_y':None,'cormod':None,'imgref':None} ### self.params['goodcycle']=None if self.params['goodrange'] is None: self.params['goodstart']=1 self.params['goodend']=len(tilts) alignerdata = apProTomo.insertAlignIteration(alignrun, protomodata, self.params, refineparamdict,ordered_imagelist[refimg]) # read tlt file print "third insert" alignmentdict, geometrydict, seriesname = apProTomo.parseTilt(self.params['tltfile']) # insert geometry model modeldata = apProTomo.insertModel2(alignerdata, geometrydict) #insert image alignments for i,imagedata in enumerate(ordered_imagelist): #Caution...assumes ordered_imagelist is in same order as tlt file apProTomo.insertTiltAlignment(alignerdata,imagedata,i,alignmentdict[i+1],center=None) print "fourth insert" apTomo.insertTiltsInAlignRun(alignrun, tiltseriesdata,tiltseriessettings,True)
def setRunDir(self): sessiondata = apDatabase.getSessionDataFromSessionName(self.params["sessionname"]) tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params["tiltseriesnumber"], sessiondata) path = self.getDefaultBaseAppionDir(sessiondata, ["tomo"]) tiltseriespath = "tiltseries%d" % self.params["tiltseriesnumber"] if self.params["full"]: tomovolumepath = "" else: tomovolumepath = self.params["volume"] intermediatepath = os.path.join(tiltseriespath, self.params["runname"], tomovolumepath) self.params["rundir"] = os.path.join(path, intermediatepath) self.params["aligndir"] = os.path.abspath(os.path.join(path, tiltseriespath, "align", self.params["runname"]))
def setRunDir(self): sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseriesnumber'],sessiondata) path = self.getDefaultBaseAppionDir(sessiondata,['tomo']) tiltseriespath = "tiltseries%d" % self.params['tiltseriesnumber'] if self.params['full']: tomovolumepath = "" else: tomovolumepath = self.params['volume'] intermediatepath = os.path.join(tiltseriespath,self.params['runname'],tomovolumepath) self.params['rundir'] = os.path.join(path,intermediatepath) self.params['aligndir'] = os.path.abspath(os.path.join(path,tiltseriespath,'align',self.params['runname']))
def setRunDir(self): sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseriesnumber'],sessiondata) path = self.getDefaultBaseAppionDir(sessiondata,['tomo']) tiltseriespath = "tiltseries%d" % self.params['tiltseriesnumber'] if self.params['full']: tomovolumepath = "" else: tomovolumepath = self.params['volume'] intermediatepath = os.path.join(tiltseriespath,self.params['runname'],tomovolumepath) self.params['rundir'] = os.path.join(path,intermediatepath) self.params['aligndir'] = os.path.abspath(os.path.join(path,tiltseriespath,'align',self.params['runname']))
def start(self): ### some of this should go in preloop functions ###do queries os.chdir(self.params['rundir']) sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata) tiltdata = apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata)
def uploadTomo(params): if not params['commit']: apDisplay.printWarning("not commiting tomogram to database") return apDisplay.printMsg("Commiting tomogram to database") sessiondata = apDatabase.getSessionDataFromSessionName(params['sessionname']) tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(params['tiltseriesnumber'],sessiondata) runname = params['runname'] name = params['name'] if params['full']: fullbin = params['bin'] else: fullbin = 1 subbin = params['bin'] alignrun = insertTomoAlignmentRun(sessiondata,None,None,None,None,fullbin,runname,params['aligndir'],'manual alignment from upload') # only tilt series in one alignrun for now insertTiltsInAlignRun(alignrun, tiltdata,None,True) alignerdata = insertAlignerParams(alignrun,params) firstimagedata = getFirstImage(tiltdata) path = os.path.abspath(params['rundir']) description = params['description'] if params['full']: thickness = params['shape'][0] * fullbin uploadfile = params['zprojfile'] projectimagedata = uploadZProjection(runname,firstimagedata,uploadfile) fullrundata = insertFullTomoRun(sessiondata,path,runname,'upload') return insertFullTomogram(sessiondata,tiltdata,alignerdata,fullrundata,name,description,projectimagedata,thickness,None,fullbin,[]) else: projectimagedata = None fulltomopath = params['rundir'].replace('/'+params['volume'],'') dummyname = 'dummy' dummydescription = 'fake full tomogram for subtomogram upload' thickness = params['shape'][0] * subbin fullrundata = insertFullTomoRun(sessiondata,fulltomopath,runname,'upload') fulltomogram = insertFullTomogram(sessiondata,tiltdata,alignerdata,fullrundata,dummyname,dummydescription,projectimagedata,thickness,None,fullbin,[]) apix = apDatabase.getPixelSize(firstimagedata) tomoq = appiondata.ApTomogramData() tomoq['session'] = sessiondata tomoq['tiltseries'] = tiltdata results = tomoq.query() index = len(results)+1 pixelsize = 1e-10 * apix * subbin runname = params['volume'] shape = map((lambda x: x * subbin), params['shape']) dimension = {'x':shape[2],'y':shape[1], 'z':shape[0]} subtomorundata = insertSubTomoRun(sessiondata, None,None,runname,params['invert'],subbin) return insertSubTomogram(fulltomogram,subtomorundata,None,0,dimension,path,name,index,pixelsize,description)
def start(self): ###do queries sessiondata = apDatabase.getSessionDataFromSessionName( self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId( self.params['tiltseries'], sessiondata) tiltseriessettings = apTomo.getTomographySettings( sessiondata, tiltseriesdata) tiltdata = apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") print "ordering list" tilts, ordered_imagelist, ordered_mrc_files, refimg = apTomo.orderImageList( tiltdata) #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int ###set up files seriesname = 'series' + str(self.params['tiltseries']) tiltfilename = seriesname + '.tlt' param_out = seriesname + '.param' maxtilt = max([abs(tilts[0]), abs(tilts[-1])]) apDisplay.printMsg("highest tilt angle is %f" % maxtilt) self.params['cos_alpha'] = math.cos(maxtilt * math.pi / 180) self.params['raw_path'] = os.path.join(self.params['rundir'], 'raw') rawexists = apParam.createDirectory(self.params['raw_path']) apDisplay.printMsg("copying raw images") newfilenames = apProTomo.getImageFiles(ordered_imagelist, self.params['raw_path'], link=False) #get alignment data alignerdata = apTomo.getAlignerdata(self.params['alignerid']) imgshape = apTomo.getTomoImageShape(ordered_imagelist[0]) imgcenter = {'x': self.imgshape[1] / 2, 'y': self.imgshape[0] / 2} specimen_euler, azimuth, origins, rotations = apTomo.getAlignmentFromDB( alignerdata, imgcenter) #write protomo2 tilt file outtltfile = 'series.tlt' seriesname = 'series' apProTomo.writeTiltFile2(outfilename, seriesname, specimen_eulers, azimuth, referenceimage)
def start(self): ### some of this should go in preloop functions ###do queries sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata) tiltdata=apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata) #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int ###set up files seriesname='series'+str(self.params['tiltseries']) tiltfilename=seriesname+'.tlt' param_out=seriesname+'.param' maxtilt=max([abs(tilts[0]),abs(tilts[-1])]) apDisplay.printMsg("highest tilt angle is %f" % maxtilt) self.params['cos_alpha']=math.cos(maxtilt*math.pi/180) self.params['raw_path']=os.path.join(self.params['rundir'],'raw') rawexists=apParam.createDirectory(self.params['raw_path']) apDisplay.printMsg("copying raw images") newfilenames=apProTomo.getImageFiles(ordered_imagelist,self.params['raw_path'], link=False) ###create tilt file #get image size from the first image imagesizex=tiltdata[0]['image'].shape[0] imagesizey=tiltdata[0]['image'].shape[1] #shift half tilt series relative to eachother #SS I'm arbitrarily making the bin parameter here 1 because it's not necessary to sample at this point shifts = apTomo.getGlobalShift(ordered_imagelist, 1, refimg) #OPTION: refinement might be more robust by doing one round of IMOD aligment to prealign images before doing protomo refine origins=apProTomo2Prep.convertShiftsToOrigin(shifts, imagesizex, imagesizey) #determine azimuth azimuth=apTomo.getAverageAzimuthFromSeries(ordered_imagelist) apProTomo2Prep.writeTileFile2(tiltfilename, seriesname, newfilenames, origins, tilts, azimuth, refimg)
def prepareTiltFile(sessionname, seriesname, tiltfilename, tiltseriesnumber, raw_path, frame_aligned_images, link=False, coarse=True): ''' Creates tlt file from basic image information and copies raw images ''' sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata) tiltdata = apTomo.getImageList([tiltseriesdata]) apDisplay.printMsg("getting imagelist") frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata) tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images) if frame_aligned_images == "True": #Azimuth is only present in the non-frame aligned images a,ordered_imagelist_for_azimuth,c,d,e = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned="False") #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int maxtilt = max([abs(tilts[0]),abs(tilts[-1])]) apDisplay.printMsg("highest tilt angle is %f" % maxtilt) if coarse == "True": if frame_aligned_images == "True": #Azimuth is only present in the non-frame aligned images azimuth = apTomo.getAverageAzimuthFromSeries(ordered_imagelist_for_azimuth) else: azimuth = apTomo.getAverageAzimuthFromSeries(ordered_imagelist) rawexists = apParam.createDirectory(raw_path) apDisplay.printMsg("Copying raw images, y-flipping, normalizing, and converting images to float32 for Protomo...") #Linking removed because raw images need to be y-flipped for Protomo:(. newfilenames, new_ordered_imagelist = apProTomo.getImageFiles(ordered_imagelist, raw_path, link=False, copy="True") ###create tilt file #get image size from the first image imagesizex = tiltdata[0]['image'].shape[1] imagesizey = tiltdata[0]['image'].shape[0] #shift half tilt series relative to eachother #SS I'm arbitrarily making the bin parameter here 1 because it's not necessary to sample at this point shifts = apTomo.getGlobalShift(ordered_imagelist, 1, refimg) #OPTION: refinement might be more robust by doing one round of IMOD aligment to prealign images before doing protomo refine origins = convertShiftsToOrigin(shifts, imagesizex, imagesizey) writeTiltFile2(tiltfilename, seriesname, newfilenames, origins, tilts, azimuth, refimg) return tilts, accumulated_dose_list, new_ordered_imagelist, maxtilt
def doseCompensate(seriesname, rundir, sessionname, tiltseriesnumber, frame_aligned_images, raw_path, pixelsize, dose_presets, dose_a, dose_b, dose_c): """ Images will be lowpass filtered using equation (3) from Grant & Grigorieff, 2015. No changes to the database are made. No backups are made. """ sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata) tiltdata = apTomo.getImageList([tiltseriesdata]) frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata) tilts, ordered_imagelist, accumulated_dose_list, ordered_mrc_files, refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned="False") if frame_aligned_images == "True": #For different image filenames a, ordered_imagelist, c, d, e = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images) newfilenames, new_ordered_imagelist = apProTomo.getImageFiles(ordered_imagelist, raw_path, link=False, copy=False) if (dose_presets == "Light"): dose_a = 0.245 dose_b = -1.6 dose_c = 12 elif (dose_presets == "Moderate"): dose_a = 0.245 dose_b = -1.665 dose_c = 2.81 elif (dose_presets == "Heavy"): dose_a = 0.245 dose_b = -1.4 dose_c = 2 apDisplay.printMsg('Dose compensating all tilt images with a=%s, b=%s, and c=%s...' % (dose_a, dose_b, dose_c)) for image, j in zip(new_ordered_imagelist, range(len(new_ordered_imagelist))): lowpass = float(np.real(complex(dose_a/(accumulated_dose_list[j] - dose_c))**(1/dose_b))) #equation (3) from Grant & Grigorieff, 2015 if lowpass < 0.0: lowpass = 0.0 im = mrc.read(image) im = imagefilter.lowPassFilter(im, apix=pixelsize, radius=lowpass, msg=False) im=imagenorm.normStdev(im) mrc.write(im, image) #Make plots apProTomo2Aligner.makeDosePlots(rundir, seriesname, tilts, accumulated_dose_list, dose_a, dose_b, dose_c) apDisplay.printMsg("Dose compensation finished for tilt-series #%s!" % tiltseriesnumber) return
def start(self): ###do queries sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params['tiltseries'],sessiondata) tiltseriessettings= apTomo.getTomographySettings(sessiondata,tiltseriesdata) tiltdata=apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") print "ordering list" tilts,ordered_imagelist,ordered_mrc_files,refimg = apTomo.orderImageList(tiltdata) #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int ###set up files seriesname='series'+str(self.params['tiltseries']) tiltfilename=seriesname+'.tlt' param_out=seriesname+'.param' maxtilt=max([abs(tilts[0]),abs(tilts[-1])]) apDisplay.printMsg("highest tilt angle is %f" % maxtilt) self.params['cos_alpha']=math.cos(maxtilt*math.pi/180) self.params['raw_path']=os.path.join(self.params['rundir'],'raw') rawexists=apParam.createDirectory(self.params['raw_path']) apDisplay.printMsg("copying raw images") newfilenames=apProTomo.getImageFiles(ordered_imagelist,self.params['raw_path'], link=False) #get alignment data alignerdata = apTomo.getAlignerdata(self.params['alignerid']) imgshape = apTomo.getTomoImageShape(ordered_imagelist[0]) imgcenter = {'x':self.imgshape[1]/2,'y':self.imgshape[0]/2} specimen_euler, azimuth, origins, rotations = apTomo.getAlignmentFromDB(alignerdata,imgcenter) #write protomo2 tilt file outtltfile='series.tlt' seriesname='series' apProTomo.writeTiltFile2(outfilename, seriesname, specimen_eulers, azimuth, referenceimage )
def getImageShapeFromTiltSeries(self): sessiondata = apDatabase.getSessionDataFromSessionName(self.params["sessionname"]) tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(self.params["tiltseriesnumber"], sessiondata) firstimagedata = apTomo.getFirstImage(tiltdata) self.imageshape = apTomo.getTomoImageShape(firstimagedata)
def ctfCorrect(seriesname, rundir, projectid, sessionname, tiltseriesnumber, tiltfilename, frame_aligned_images, pixelsize, DefocusTol, iWidth, amp_contrast): """ Leginondb will be queried to get the 'best' defocus estimate on a per-image basis. Confident defoci will be gathered and unconfident defoci will be interpolated. Images will be CTF corrected by phase flipping using ctfphaseflip from the IMOD package. A plot of the defocus values will is made. A CTF plot using the mean defocus is made. """ try: apDisplay.printMsg('CTF correcting all tilt images using defocus values from Leginon database...') os.chdir(rundir) raw_path=rundir+'/raw/' ctfdir='%s/ctf_correction/' % rundir os.system("mkdir %s" % ctfdir) defocus_file_full=ctfdir+seriesname+'_defocus.txt' tilt_file_full=ctfdir+seriesname+'_tilts.txt' image_list_full=ctfdir+seriesname+'_images.txt' uncorrected_stack=ctfdir+'stack_uncorrected.mrc' corrected_stack=ctfdir+'stack_corrected.mrc' out_full=ctfdir+'out' log_file_full=ctfdir+'ctf_correction.log' project='ap'+projectid sinedon.setConfig('appiondata', db=project) sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata) tiltdata = apTomo.getImageList([tiltseriesdata]) frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata) tilts,ordered_imagelist,accumulated_dose_list,ordered_mrc_files,refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images) cs = tiltdata[0]['scope']['tem']['cs']*1000 voltage = int(tiltdata[0]['scope']['high tension']/1000) if os.path.isfile(ctfdir+'out/out01.mrc'): #Throw exception if already ctf corrected sys.exit() #Get tilt azimuth cmd="awk '/TILT AZIMUTH/{print $3}' %s" % (tiltfilename) proc=subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) (tilt_azimuth, err) = proc.communicate() tilt_azimuth=float(tilt_azimuth) estimated_defocus=[] for image in range(len(ordered_imagelist)): imgctfdata=ctfdb.getBestCtfValue(ordered_imagelist[image], msg=False) try: if imgctfdata['resolution_50_percent'] < 100.0: #if there's a yellow ring in Appion, trust defocus estimation estimated_defocus.append((imgctfdata['defocus1']+imgctfdata['defocus2'])*1000000000/2) else: #Poorly estimated. Guess its value later estimated_defocus.append(999999999) except: #No data. Guess its value later estimated_defocus.append(999999999) #Find mean and stdev to prune out confident defocus values that are way off defocus_stats_list=filter(lambda a: a != 999999999, estimated_defocus) avg=np.array(defocus_stats_list).mean() stdev=np.array(defocus_stats_list).std() good_tilts=[] good_defocus_list=[] for tilt, defocus in zip(tilts, estimated_defocus): if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev): good_defocus_list.append(defocus) good_tilts.append(tilt) #Using a linear best fit because quadratic and cubic go off the rails. Estimation doesn't need to be extremely accurate anyways. x=np.linspace(int(round(tilts[0])), int(round(tilts[len(tilts)-1])), 1000) s=scipy.interpolate.UnivariateSpline(good_tilts,good_defocus_list,k=1) y=s(x) #Make defocus list with good values and interpolations for bad values finished_defocus_list=[] for tilt, defocus in zip(tilts, estimated_defocus): if (defocus != 999999999) and (defocus < avg + stdev) and (defocus > avg - stdev): finished_defocus_list.append(int(round(defocus))) else: #Interpolate finished_defocus_list.append(int(round(y[int(round(tilt))]))) new_avg=np.array(finished_defocus_list).mean() new_stdev=np.array(finished_defocus_list).std() #Write defocus file, tilt file, and image list file for ctfphaseflip and newstack f = open(defocus_file_full,'w') f.write("%d\t%d\t%.2f\t%.2f\t%d\t2\n" % (1,1,tilts[0],tilts[0],finished_defocus_list[0])) for i in range(1,len(tilts)): f.write("%d\t%d\t%.2f\t%.2f\t%d\n" % (i+1,i+1,tilts[i],tilts[i],finished_defocus_list[i])) f.close() f = open(tilt_file_full,'w') for tilt in tilts: f.write("%.2f\n" % tilt) f.close() mrc_list=[] presetname=tiltdata[0]['preset']['name'] for image in ordered_mrc_files: mrcname=presetname+image.split(presetname)[-1] mrc_list.append(raw_path+'/'+mrcname) f = open(image_list_full,'w') f.write("%d\n" % len(tilts)) for filename in mrc_list: f.write(filename+'\n') f.write("%d\n" % 0) f.close() #Rotate and pad images so that they are treated properly by ctfphaseflip. apDisplay.printMsg("Preparing images for IMOD...") for filename in mrc_list: image=mrc.read(filename) dimx=len(image[0]) dimy=len(image) #First rotate 90 degrees in counter-clockwise direction. This makes it so positive angle images are higher defocused on the right side of the image image=np.rot90(image, k=-1) #Rotate image and write image=imrotate(image, -tilt_azimuth, order=1) #Linear interpolation is fastest and there is barely a difference between linear and cubic mrc.write(image, filename) f = open(log_file_full,'w') #Make stack for correction,phase flip, extract images, replace images cmd1="newstack -fileinlist %s -output %s > %s" % (image_list_full, uncorrected_stack, log_file_full) f.write("%s\n\n" % cmd1) print cmd1 subprocess.check_call([cmd1], shell=True) cmd2="ctfphaseflip -input %s -output %s -AngleFile %s -defFn %s -pixelSize %s -volt %s -DefocusTol %s -iWidth %s -SphericalAberration %s -AmplitudeContrast %s 2>&1 | tee %s" % (uncorrected_stack, corrected_stack, tilt_file_full, defocus_file_full, pixelsize/10, voltage, DefocusTol, iWidth, cs, amp_contrast, log_file_full) f.write("\n\n%s\n\n" % cmd2) print cmd2 subprocess.check_call([cmd2], shell=True) cmd3="newstack -split 1 -append mrc %s %s >> %s" % (corrected_stack, out_full, log_file_full) f.write("\n\n%s\n\n" % cmd3) print cmd3 subprocess.check_call([cmd3], shell=True) f.write("\n\n") apDisplay.printMsg("Overwriting uncorrected raw images with CTF corrected images") new_images=glob.glob(ctfdir+'out*mrc') new_images.sort() #Unrotate and unpad images for filename in new_images: image=mrc.read(filename) image=imrotate(image, tilt_azimuth, order=1) image=np.rot90(image, k=1) big_dimx=len(image[0]) big_dimy=len(image) cropx1=int((big_dimx-dimx)/2) cropx2=int(dimx+(big_dimx-dimx)/2) cropy1=int((big_dimy-dimy)/2) cropy2=int(dimy+(big_dimy-dimy)/2) image=image[cropy1:cropy2,cropx1:cropx2] mrc.write(image, filename) for i in range(len(new_images)): cmd4="rm %s; ln %s %s" % (mrc_list[i], new_images[i], mrc_list[i]) f.write("%s\n" % cmd4) os.system(cmd4) #Make plots apProTomo2Aligner.makeDefocusPlot(rundir, seriesname, defocus_file_full) apProTomo2Aligner.makeCTFPlot(rundir, seriesname, defocus_file_full, voltage, cs) cleanup="rm %s %s" % (uncorrected_stack, corrected_stack) os.system(cleanup) output1="%.2f%% of the images for tilt-series #%s had poor defocus estimates or fell outside of one standard deviation from the original mean." % (100*(len(estimated_defocus)-len(defocus_stats_list))/len(estimated_defocus), tiltseriesnumber) output2="The defocus mean and standard deviation for tilt-series #%s after interpolating poor values is %.2f and %.2f microns, respectively." % (tiltseriesnumber, new_avg/1000, new_stdev/1000) f.write("\n");f.write(output1);f.write("\n");f.write(output2);f.write("\n");f.close() apDisplay.printMsg(output1) apDisplay.printMsg(output2) apDisplay.printMsg("CTF correction finished for tilt-series #%s!" % tiltseriesnumber) except subprocess.CalledProcessError: apDisplay.printError("An IMOD command failed, so CTF correction could not be completed. Make sure IMOD is in your $PATH.") except SystemExit: apDisplay.printWarning("It looks like you've already CTF corrected tilt-series #%s. Skipping CTF correction!" % tiltseriesnumber) except: apDisplay.printError("CTF correction could not be completed. Make sure IMOD, numpy, and scipy are in your $PATH. Make sure defocus has been estimated through Appion.\n")
def start(self): ### some of this should go in preloop functions ###do queries sessiondata = apDatabase.getSessionDataFromSessionName( self.params['sessionname']) self.sessiondata = sessiondata tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId( self.params['tiltseries'], sessiondata) tiltseriessettings = apTomo.getTomographySettings( sessiondata, tiltseriesdata) tiltdata = apTomo.getImageList([tiltseriesdata]) description = self.params['description'] apDisplay.printMsg("getting imagelist") print "ordering list" tilts, ordered_imagelist, ordered_mrc_files, refimg = apTomo.orderImageList( tiltdata) #tilts are tilt angles, ordered_imagelist are imagedata, ordered_mrc_files are paths to files, refimg is an int ###set up files seriesname = self.params['seriesname'] print type(seriesname) # param_out=seriesname+'.param' ###insert protomo run params print "first insert" protomodata = apProTomo.insertProtomoParams(seriesname) print "second insert" alignrun = apTomo.insertTomoAlignmentRun(sessiondata, tiltseriessettings, None, protomodata, None, 1, self.params['runname'], self.params['rundir'], self.params['description']) ###insert protomo alignment ###hack to get around need to parse protomo param file #should read imgref from tlt file refineparamdict = { 'alismp': None, 'alibox_x': None, 'alibox_y': None, 'cormod': None, 'imgref': None } ### self.params['goodcycle'] = None if self.params['goodrange'] is None: self.params['goodstart'] = 1 self.params['goodend'] = len(tilts) alignerdata = apProTomo.insertAlignIteration(alignrun, protomodata, self.params, refineparamdict, ordered_imagelist[refimg]) # read tlt file print "third insert" alignmentdict, geometrydict, seriesname = apProTomo.parseTilt( self.params['tltfile']) # insert geometry model modeldata = apProTomo.insertModel2(alignerdata, geometrydict) #insert image alignments for i, imagedata in enumerate(ordered_imagelist): #Caution...assumes ordered_imagelist is in same order as tlt file apProTomo.insertTiltAlignment(alignerdata, imagedata, i, alignmentdict[i + 1], center=None) print "fourth insert" apTomo.insertTiltsInAlignRun(alignrun, tiltseriesdata, tiltseriessettings, True)
def uploadTomo(params): if not params["commit"]: apDisplay.printWarning("not commiting tomogram to database") return apDisplay.printMsg("Commiting tomogram to database") sessiondata = apDatabase.getSessionDataFromSessionName(params["sessionname"]) tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(params["tiltseriesnumber"], sessiondata) runname = params["runname"] name = params["name"] if params["full"]: fullbin = params["bin"] else: fullbin = 1 subbin = params["bin"] alignrun = insertTomoAlignmentRun( sessiondata, None, None, None, None, fullbin, runname, params["aligndir"], "manual alignment from upload" ) # only tilt series in one alignrun for now insertTiltsInAlignRun(alignrun, tiltdata, None, True) alignerdata = insertAlignerParams(alignrun, params) firstimagedata = getFirstImage(tiltdata) path = os.path.abspath(params["rundir"]) description = params["description"] if params["full"]: thickness = params["shape"][0] * fullbin uploadfile = params["zprojfile"] projectimagedata = uploadZProjection(runname, firstimagedata, uploadfile) fullrundata = insertFullTomoRun(sessiondata, path, runname, "upload") return insertFullTomogram( sessiondata, tiltdata, alignerdata, fullrundata, name, description, projectimagedata, thickness, None, fullbin, [], ) else: projectimagedata = None fulltomopath = params["rundir"].replace("/" + params["volume"], "") dummyname = "dummy" dummydescription = "fake full tomogram for subtomogram upload" thickness = params["shape"][0] * subbin fullrundata = insertFullTomoRun(sessiondata, fulltomopath, runname, "upload") fulltomogram = insertFullTomogram( sessiondata, tiltdata, alignerdata, fullrundata, dummyname, dummydescription, projectimagedata, thickness, None, fullbin, [], ) apix = apDatabase.getPixelSize(firstimagedata) tomoq = appiondata.ApTomogramData() tomoq["session"] = sessiondata tomoq["tiltseries"] = tiltdata results = tomoq.query() index = len(results) + 1 pixelsize = 1e-10 * apix * subbin runname = params["volume"] shape = map((lambda x: x * subbin), params["shape"]) dimension = {"x": shape[2], "y": shape[1], "z": shape[0]} subtomorundata = insertSubTomoRun(sessiondata, None, None, runname, params["invert"], subbin) return insertSubTomogram( fulltomogram, subtomorundata, None, 0, dimension, path, name, index, pixelsize, description )
def uploadTomo(params): if not params['commit']: apDisplay.printWarning("not commiting tomogram to database") return apDisplay.printMsg("Commiting tomogram to database") sessiondata = apDatabase.getSessionDataFromSessionName( params['sessionname']) tiltdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId( params['tiltseriesnumber'], sessiondata) runname = params['runname'] name = params['name'] if params['full']: fullbin = params['bin'] else: fullbin = 1 subbin = params['bin'] alignrun = insertTomoAlignmentRun(sessiondata, None, None, None, None, fullbin, runname, params['aligndir'], 'manual alignment from upload') # only tilt series in one alignrun for now insertTiltsInAlignRun(alignrun, tiltdata, None, True) alignerdata = insertAlignerParams(alignrun, params) firstimagedata = getFirstImage(tiltdata) path = os.path.abspath(params['rundir']) description = params['description'] if params['full']: thickness = params['shape'][0] * fullbin uploadfile = params['zprojfile'] projectimagedata = uploadZProjection(runname, firstimagedata, uploadfile) fullrundata = insertFullTomoRun(sessiondata, path, runname, 'upload') return insertFullTomogram(sessiondata, tiltdata, alignerdata, fullrundata, name, description, projectimagedata, thickness, None, fullbin, []) else: projectimagedata = None fulltomopath = params['rundir'].replace('/' + params['volume'], '') dummyname = 'dummy' dummydescription = 'fake full tomogram for subtomogram upload' thickness = params['shape'][0] * subbin fullrundata = insertFullTomoRun(sessiondata, fulltomopath, runname, 'upload') fulltomogram = insertFullTomogram(sessiondata, tiltdata, alignerdata, fullrundata, dummyname, dummydescription, projectimagedata, thickness, None, fullbin, []) apix = apDatabase.getPixelSize(firstimagedata) tomoq = appiondata.ApTomogramData() tomoq['session'] = sessiondata tomoq['tiltseries'] = tiltdata results = tomoq.query() index = len(results) + 1 pixelsize = 1e-10 * apix * subbin runname = params['volume'] shape = map((lambda x: x * subbin), params['shape']) dimension = {'x': shape[2], 'y': shape[1], 'z': shape[0]} subtomorundata = insertSubTomoRun(sessiondata, None, None, runname, params['invert'], subbin) return insertSubTomogram(fulltomogram, subtomorundata, None, 0, dimension, path, name, index, pixelsize, description)