def insertRefineParticleData(self, iterdata, parttree): apDisplay.printMsg("Inserting particle data") phase_threshold = float(iterdata['frealignParams']['thresh']) count = 0 for partdict in parttree: count += 1 if count % 1000 == 0: apDisplay.printMsg("Inserted %d particles"%(count)) partrefineq = appiondata.ApRefineParticleData() partrefineq['refineIter'] = iterdata partnum = partdict['partnum'] stackpartid = self.stackmapping[partnum] stackpartdata = appiondata.ApStackParticleData.direct_query(stackpartid) partrefineq['particle'] = stackpartdata partkeys = ('shiftx', 'shifty', 'euler1', 'euler2', 'euler3', 'phase_residual') for key in partkeys: partrefineq[key] = partdict[key] ### check if particle was rejected if partdict['phase_residual'] > iterdata['frealignParams']['thresh']: partrefineq['refine_keep'] = False else: partrefineq['refine_keep'] = True partrefineq['mirror'] = False partrefineq['postRefine_keep'] = False partrefineq['euler_convention'] = 'zyz' if self.params['commit'] is True: partrefineq.insert() return
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['description'] is None: apDisplay.printError("run description was not defined") if self.params['templatelist'] is None: apDisplay.printError("template list was not provided") if self.params['lastring'] is None: apDisplay.printError("a last ring radius was not provided") if self.params['runname'] is None: apDisplay.printError("run name was not defined") stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid'])/self.params['bin'] if self.params['lastring'] > boxsize/2-2: apDisplay.printError("last ring radius is too big for boxsize " +str(self.params['lastring'])+" > "+str(boxsize/2-2)) if self.params['lastring']+self.params['xysearch'] > boxsize/2-2: apDisplay.printError("last ring plus xysearch radius is too big for boxsize " +str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2)) ### convert / check template data self.templatelist = self.params['templatelist'].strip().split(",") if not self.templatelist or type(self.templatelist) != type([]): apDisplay.printError("could not parse template list="+self.params['templatelist']) self.params['numtemplate'] = len(self.templatelist) apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates")
def boxerRotate(imgfile, parttree, outstack, boxsize): """ boxes the particles with expanded size, applies a rotation to particle, reduces boxsize to requested size, and saves them to a imagic file """ # size needed is sqrt(2)*boxsize, using 1.5 to be extra safe bigboxsize = int(math.ceil(1.5*boxsize)) imgarray = mrc.read(imgfile) bigboxedparticles = boxerMemory(imgarray, parttree, bigboxsize) boxedparticles = [] boxshape = (boxsize,boxsize) apDisplay.printMsg("Rotating particles...") for i in range(len(bigboxedparticles)): if i % 10 == 0: sys.stderr.write(".") bigboxpart = bigboxedparticles[i] partdict = parttree[i] ### add 90 degrees because database angle is from x-axis not y-axis angle = partdict['angle']+90.0 rotatepart = ndimage.rotate(bigboxpart, angle=angle, reshape=False, order=1) boxpart = imagefilter.frame_cut(rotatepart, boxshape) boxedparticles.append(boxpart) sys.stderr.write("done\n") apImagicFile.writeImagic(boxedparticles, outstack) return True
def phaseFlipWholeImage(self, inimgpath, imgdata): outimgpath = os.path.join(self.params['rundir'], self.shortname+"-ctfcorrect.dwn.mrc") ### High tension on CM is given in kv instead of v so do not divide by 1000 in that case if imgdata['scope']['tem']['name'] == "CM": voltage = imgdata['scope']['high tension'] else: voltage = (imgdata['scope']['high tension'])/1000 apix = apDatabase.getPixelSize(imgdata) defocus, ampconst = self.getDefocusAmpConstForImage(imgdata, True) defocus *= 1.0e6 self.checkDefocus(defocus, self.shortname) ### get all CTF parameters, we also need to get the CS value from the database ctfdata = self.getBestCtfValue(imgdata, False) #ampconst = ctfdata['amplitude_contrast'] ### we could use this too # find cs cs = self.getCS(ctfdata) parmstr = ("parm=%f,200,1,%.3f,0,17.4,9,1.53,%i,%.1f,%f" %(defocus, ampconst, voltage, cs, apix)) emancmd = ("applyctf %s %s %s setparm flipphase" % (inimgpath, outimgpath, parmstr)) apDisplay.printMsg("phaseflipping entire micrograph with defocus "+str(round(defocus,3))+" microns") apEMAN.executeEmanCmd(emancmd, showcmd = True) return outimgpath
def processParticles(self, imgdata, partdatas, shiftdata): self.shortname = apDisplay.short(imgdata['filename']) ### if only selected points along helix, ### fill in points with helical step if self.params['helicalstep']: apix = apDatabase.getPixelSize(imgdata) partdatas = self.fillWithHelicalStep(partdatas, apix) ### run batchboxer self.boxedpartdatas, self.imgstackfile, self.partmeantree = self.boxParticlesFromImage(imgdata, partdatas, shiftdata) if self.boxedpartdatas is None: self.stats['lastpeaks'] = 0 apDisplay.printWarning("no particles were boxed from "+self.shortname+"\n") self.badprocess = True return None self.stats['lastpeaks'] = len(self.boxedpartdatas) apDisplay.printMsg("do not break function now otherwise it will corrupt stack") #time.sleep(1.0) ### merge image particles into big stack totalpart = self.mergeImageStackIntoBigStack(self.imgstackfile, imgdata) ### create a stack average every so often if self.stats['lastpeaks'] > 0: totalPartices = self.existingParticleNumber+self.stats['peaksum']+self.stats['lastpeaks'] logpeaks = math.log(totalPartices) if logpeaks > self.logpeaks: self.logpeaks = math.ceil(logpeaks) numpeaks = math.ceil(math.exp(self.logpeaks)) apDisplay.printMsg("writing averaging stack, next average at %d particles"%(numpeaks)) mrc.write(self.summedParticles/float(totalPartices), "average.mrc") return totalpart
def checkConflicts(self): ### setup correct database after we have read the project id if 'projectid' in self.params and self.params['projectid'] is not None: apDisplay.printMsg("Using split database") # use a project database newdbname = apProject.getAppionDBFromProjectId(self.params['projectid']) sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '"+newdbname+"'", "green") # DD processes self.dd = apDDprocess.DDStackProcessing() print self.dd # get stack data self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid']) self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True) self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) # query image qimage = self.stackparts[0]['particle']['image'] # DD info self.dd.setImageData(qimage) self.dd.setDDStackRun(self.params['ddstackid']) self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def boxParticlesFromImage(self, imgdata, partdatas, shiftdata): ### convert database particle data to coordinates and write boxfile boxfile = os.path.join(self.params['rundir'], imgdata['filename']+".box") parttree, boxedpartdatas = apBoxer.processParticleData(imgdata, self.boxsize, partdatas, shiftdata, boxfile, rotate=self.params['rotate']) if self.params['boxfiles']: ### quit and return, boxfile created, now process next image return None, None, None ### check if we have particles again if len(partdatas) == 0 or len(parttree) == 0: apDisplay.printColor(self.shortname+" has no remaining particles and has been rejected\n","cyan") return None, None, None ### set up output file path imgstackfile = os.path.join(self.params['rundir'], self.shortname+".hed") if (self.is_dd_stack and (self.params['nframe'] or self.params['driftlimit']) and not self.params['phaseflipped'] and not self.params['rotate']): # If processing on whole image is not needed, it is more efficient to use mmap to box frame stack apDisplay.printMsg("boxing "+str(len(parttree))+" particles into temp file: "+imgstackfile) framelist = self.dd.getFrameList(self.params) apBoxer.boxerFrameStack(self.dd.framestackpath, parttree, imgstackfile, self.boxsize, framelist) else: self._boxParticlesFromImage(imgdata, parttree, imgstackfile) partmeantree = self.calculateParticleStackStats(imgstackfile, boxedpartdatas) imgstackfile = self.postProcessParticleStack(imgdata, imgstackfile, boxedpartdatas, len(parttree)) return boxedpartdatas, imgstackfile, partmeantree
def importPicks(self, picks1, picks2, tight=False, msg=True): t0 = time.time() #print picks1 #print self.currentpicks1 curpicks1 = numpy.asarray(self.currentpicks1) curpicks2 = numpy.asarray(self.currentpicks2) #print curpicks1 # get picks apTiltTransform.setPointsFromArrays(curpicks1, curpicks2, self.data) pixdiam = self.data['pixdiam'] if tight is True: pixdiam /= 4.0 #print self.data, pixdiam list1, list2 = apTiltTransform.alignPicks2(picks1, picks2, self.data, limit=pixdiam, msg=msg) if list1.shape[0] == 0 or list2.shape[0] == 0: apDisplay.printWarning("No new picks were found") # merge picks newpicks1, newpicks2 = apTiltTransform.betterMergePicks(curpicks1, list1, curpicks2, list2, msg=msg) newparts = newpicks1.shape[0] - curpicks1.shape[0] # copy over picks self.currentpicks1 = newpicks1 self.currentpicks2 = newpicks2 if msg is True: apDisplay.printMsg("Inserted "+str(newparts)+" new particles in "+apDisplay.timeString(time.time()-t0)) return True
def optimizeAngles(self, msg=True): t0 = time.time() ### run find theta na1 = numpy.array(self.currentpicks1, dtype=numpy.int32) na2 = numpy.array(self.currentpicks2, dtype=numpy.int32) fittheta = radermacher.tiltang(na1, na2) if not fittheta or not 'wtheta' in fittheta: return theta = fittheta['wtheta'] thetadev = fittheta['wthetadev'] if msg is True: thetastr = ("%3.3f +/- %2.2f" % (theta, thetadev)) tristr = apDisplay.orderOfMag(fittheta['numtri'])+" of "+apDisplay.orderOfMag(fittheta['tottri']) tristr = (" (%3.1f " % (100.0 * fittheta['numtri'] / float(fittheta['tottri'])))+"%) " apDisplay.printMsg("Tilt angle "+thetastr+tristr) self.data['theta'] = fittheta['wtheta'] ### run optimize angles lastiter = [80,80,80] count = 0 totaliter = 0 while max(lastiter) > 75 and count < 30: count += 1 lsfit = self.runLeastSquares() lastiter[2] = lastiter[1] lastiter[1] = lastiter[0] lastiter[0] = lsfit['iter'] totaliter += lsfit['iter'] if msg is True: apDisplay.printMsg("Least Squares: "+str(count)+" rounds, "+str(totaliter) +" iters, rmsd of "+str(round(lsfit['rmsd'],4))+" pixels in "+apDisplay.timeString(time.time()-t0)) return
def commitToDatabase(self, imgdata): """ Uses the appionLoop commit """ imgassess = apDatabase.getImgCompleteStatus(imgdata) tiltdata = apTiltPair.getTiltPair(imgdata) msg = not self.params['background'] if tiltdata is None: if imgassess is not False: apDisplay.printColor("\nrejecting unpaired image: "+apDisplay.short(imgdata['filename']), "red") apDatabase.insertImgAssessmentStatus(imgdata, self.params['runid'], False, msg=msg) self.reject+=1 return if self.params['background'] is False: apDisplay.printMsg("tiltpair: "+apDisplay.short(tiltdata['filename'])) tiltassess = apDatabase.getImgCompleteStatus(tiltdata) if imgassess is False or tiltassess is False: if imgassess is not False: apDisplay.printColor("\nrejecting bad tilt images: "+apDisplay.short(imgdata['filename']), "magenta") if tiltassess is not False: apDisplay.printColor("\nrejecting bad tilt images: "+apDisplay.short(tiltdata['filename']), "magenta") apDatabase.insertImgAssessmentStatus(imgdata, self.params['runid'], False, msg=msg) apDatabase.insertImgAssessmentStatus(tiltdata, self.params['runid'], False, msg=msg) self.reject+=2 if self.params['background'] is False: print "Assessment:", imgassess, tiltassess return
def getOverlap(self, image1, image2, msg=True): t0 = time.time() bestOverlap, tiltOverlap = apTiltTransform.getOverlapPercent(image1, image2, self.data) overlapStr = str(round(100*bestOverlap,2))+"% and "+str(round(100*tiltOverlap,2))+"%" if msg is True: apDisplay.printMsg("Found overlaps of "+overlapStr+" in "+apDisplay.timeString(time.time()-t0)) self.data['overlap'] = bestOverlap
def convertStackToSpider(self, emanstackfile, classnum): """ takes the stack file and creates a spider file ready for processing """ if not os.path.isfile(emanstackfile): apDisplay.printError("stackfile does not exist: "+emanstackfile) ### first high pass filter particles apDisplay.printMsg("pre-filtering particles") apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid']) emancmd = ("proc2d "+emanstackfile+" "+emanstackfile +" apix="+str(apix)+" hp="+str(self.params['highpasspart']) +" inplace") apEMAN.executeEmanCmd(emancmd, verbose=True) ### convert imagic stack to spider emancmd = "proc2d " emancmd += emanstackfile+" " spiderstack = os.path.join(self.params['rundir'], str(classnum), "otrstack"+self.timestamp+".spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack+" " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan") return spiderstack
def convertSQLtoEulerTree(self, results): t0 = time.time() eulertree = [] for row in results: if len(row) < 11: apDisplay.printError("delete MySQL cache file and run again") try: eulerpair = { 'part1': {}, 'part2': {} } eulerpair['part1']['partid'] = int(row[0]) eulerpair['part1']['dbid'] = int(row[1]) eulerpair['part1']['euler1'] = float(row[2]) eulerpair['part1']['euler2'] = float(row[3]) eulerpair['part1']['euler3'] = float(row[4]) eulerpair['part1']['mirror'] = self.nullOrValue(row[5]) eulerpair['part1']['reject'] = not self.nullOrValue(row[6]) eulerpair['part1']['tilt'] = apStack.getStackParticleTilt(eulerpair['part1']['dbid']) eulerpair['part2']['partid'] = int(row[7]) eulerpair['part2']['dbid'] = int(row[8]) eulerpair['part2']['euler1'] = float(row[9]) eulerpair['part2']['euler2'] = float(row[10]) eulerpair['part2']['euler3'] = float(row[11]) eulerpair['part2']['mirror'] = self.nullOrValue(row[12]) eulerpair['part2']['reject'] = not self.nullOrValue(row[13]) eulerpair['part2']['tilt'] = apStack.getStackParticleTilt(eulerpair['part2']['dbid']) eulertree.append(eulerpair) except: print row apDisplay.printError("bad row entry") apDisplay.printMsg("Converted "+str(len(eulertree))+" eulers in "+apDisplay.timeString(time.time()-t0)) return eulertree
def xmippNormStack(self, inStackPath, outStackPath): ### convert stack into single spider files selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath) ### setup Xmipp command xmippexe = apParam.getExecPath("xmipp_normalize", die=True) apDisplay.printMsg("Using Xmipp to normalize particle stack") normtime = time.time() xmippopts = ( " " +" -i %s"%os.path.join(self.params['rundir'],selfile) +" -method Ramp " +" -background circle %i"%(self.stack['boxsize']/self.params['bin']*0.4) +" -remove_black_dust" +" -remove_white_dust" +" -thr_black_dust -%.2f"%(self.params['xmipp-norm']) +" -thr_white_dust %.2f"%(self.params['xmipp-norm']) ) xmippcmd = xmippexe+" "+xmippopts apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True) normtime = time.time() - normtime apDisplay.printMsg("Xmipp normalization time: "+apDisplay.timeString(normtime)) ### recombine particles to a single imagic stack tmpstack = "tmp.xmippStack.hed" apXmipp.gatherSingleFilesIntoStack(selfile,tmpstack) apFile.moveStack(tmpstack,outStackPath) ### clean up directory apFile.removeFile(selfile) apFile.removeDir("partfiles")
def writeRawDataFile(self, eulertree): #write to file rawfile = "rawdata"+self.datastr+".dat" apDisplay.printMsg("Writing raw data to file: "+rawfile) r = open(rawfile, "w") r.write("p1-id\tp1-e1\tp1-e2\tp1-e3\tmirror\treject\t" +"p2-id\tp2-e1\tp2-e2\tp2-e3\tmirror\treject\t" +"ang-dist\trot-dist\ttotal-dist\n") for eulerpair in eulertree: mystr = ( str(eulerpair['part1']['partid'])+"\t"+ str(round(eulerpair['part1']['euler1'],2))+"\t"+ str(round(eulerpair['part1']['euler2'],2))+"\t"+ str(round(eulerpair['part1']['euler3'],2))+"\t"+ str(eulerpair['part1']['mirror'])+"\t"+ str(eulerpair['part1']['reject'])+"\t"+ str(eulerpair['part2']['partid'])+"\t"+ str(round(eulerpair['part2']['euler1'],2))+"\t"+ str(round(eulerpair['part2']['euler2'],2))+"\t"+ str(round(eulerpair['part2']['euler3'],2))+"\t"+ str(eulerpair['part2']['mirror'])+"\t"+ str(eulerpair['part2']['reject'])+"\t"+ #str(round(eulerpair['angdist'],2))+"\t"+ str(round(eulerpair['rotdist'],2))+"\t"+ str(round(eulerpair['totdist'],2))+"\n" ) r.write(mystr) r.close() return
def removeOverlappingPeaks(peaktree, cutoff, msg=True, doubles=False): #distance in pixels for two peaks to be too close together if msg is True: apDisplay.printMsg("overlap distance cutoff: "+str(round(cutoff,1))+" pixels") cutsq = cutoff**2 + 1 initpeaks = len(peaktree) #orders peaks from smallest to biggest peaktree.sort(_peakCompareSmallBig) doublepeaktree = [] i=0 while i < len(peaktree): j = i+1 while j < len(peaktree): distsq = peakDistSq(peaktree[i], peaktree[j]) if(distsq < cutsq): doublepeaktree.append(peaktree[j]) del peaktree[i] i -= 1 j = len(peaktree) j += 1 i += 1 if doubles is True: peaktree = removeOverlappingPeaks(doublepeaktree, cutoff, False, False) numpeaks = len(peaktree) if msg is True: apDisplay.printMsg("kept "+str(numpeaks)+" non-overlapping peaks of " +str(initpeaks)+" total peaks") return peaktree
def findPeaks(imgdict, maplist, params, maptype="ccmaxmap", pikfile=True): peaktreelist = [] count = 0 imgname = imgdict['filename'] mapdir = os.path.join(params['rundir'], "maps") thresh = float(params["thresh"]) bin = int(params["bin"]) diam = float(params["diam"]) apix = float(params["apix"]) olapmult = float(params["overlapmult"]) maxpeaks = int(params["maxpeaks"]) maxthresh = params["maxthresh"] maxsizemult = float(params["maxsize"]) peaktype = params["peaktype"] msg = not params['background'] pixdiam = diam/apix/float(bin) pixrad = diam/apix/2.0/float(bin) tmpldbid = None mapdiam = None for imgmap in maplist: count += 1 if 'templateIds' in params: #template correlator tmpldbid = params['templateIds'][count-1] elif 'diamarray' in params: #dogpicker mapdiam = params['diamarray'][count-1] #find peaks peaktree = findPeaksInMap(imgmap, thresh, pixdiam, count, olapmult, maxpeaks, maxsizemult, msg, tmpldbid, mapdiam, bin=bin, peaktype=peaktype) #remove border peaks peaktree = removeBorderPeaks(peaktree, pixdiam, imgdict['image'].shape[1], imgdict['image'].shape[0]) #write map to jpeg with highlighted peaks outfile = os.path.join(mapdir, imgname+"."+maptype+str(count)+".jpg") createPeakMapImage(peaktree, imgmap, outfile, pixrad, bin, msg) #write pikfile if pikfile is True: peakTreeToPikFile(peaktree, imgname, count, params['rundir']) #append to complete list of peaks peaktreelist.append(peaktree) peaktree = mergePeakTrees(imgdict, peaktreelist, params, msg, pikfile=pikfile) #max threshold if maxthresh is not None: precount = len(peaktree) peaktree = maxThreshPeaks(peaktree, maxthresh) postcount = len(peaktree) #if precount != postcount: apDisplay.printMsg("Filtered %d particles above threshold %.2f"%(precount-postcount,maxthresh)) return peaktree
def getNumAlignedParticles(self): t0 = time.time() self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid']) oldalignedstack = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['imagicfile']) numpart = apFile.numImagesInStack(oldalignedstack) apDisplay.printMsg("numpart="+str(numpart)+" in "+apDisplay.timeString(time.time()-t0)) return numpart
def fillSimilarityMatrix(self, alignedstack): ### Get initial correlation values ### this is really, really slow similarfile = "similarities.dat" simstack = similarityStack() simstack.similarfile = similarfile simstack.start(alignedstack) if not os.path.isfile(similarfile): apDisplay.printError("Failed to create similarity file") simf = open(similarfile, 'r') simlist = [] count = 0 for line in simf: count += 1 sline = line.strip() slist = sline.split() ccval = float(slist[2]) simlist.append(ccval) simf.close() apDisplay.printMsg("There are %d lines in the sim file: %s"%(count, similarfile)) numpart = apFile.numImagesInStack(alignedstack) if count != numpart*(numpart-1): ### we have a valid file already apDisplay.printError("There are only %d lines need to have %d"%(count, numpart*(numpart-1))) return similarfile, simlist
def calcResolution(self, partlist, stackfile, apix): ### group particles by refnum reflistsdict = {} for partdict in partlist: refnum = partdict['template'] partnum = partdict['num'] if not refnum in reflistsdict: reflistsdict[refnum] = [] reflistsdict[refnum].append(partnum) ### get resolution self.resdict = {} boxsizetuple = apFile.getBoxSize(stackfile) boxsize = boxsizetuple[0] for refnum in reflistsdict.keys(): partlist = reflistsdict[refnum] esttime = 3e-6 * len(partlist) * boxsize**2 apDisplay.printMsg("Ref num %d; %d parts; est time %s" %(refnum, len(partlist), apDisplay.timeString(esttime))) frcdata = apFourier.spectralSNRStack(stackfile, apix, partlist, msg=False) frcfile = "frcplot-%03d.dat"%(refnum) apFourier.writeFrcPlot(frcfile, frcdata, apix, boxsize) res = apFourier.getResolution(frcdata, apix, boxsize) self.resdict[refnum] = res return
def parseParticleDataIterationFile(self,paramfile,test=False): ''' parse data.star file from Relion 1.2: ''' if not os.path.isfile(paramfile): apDisplay.printError("Relion data.star file does not exist: %s" % (paramfile)) apDisplay.printMsg("Parsing parameter file: %s" % (paramfile)) # Use the star file class to read the parameter file f = starFile.StarFile( paramfile ) f.read() dataBlock = f.getDataBlock("data_images") loopDict = dataBlock.getLoopDict() # there is only one loop in the data_images block partnum = 1 # partnum starts with 1, not 0 partdict = {} for valueSet in loopDict: paramdict = { 'partnum' : partnum, 'angleRot' : float(valueSet["_rlnAngleRot"]), 'angleTilt' : float(valueSet["_rlnAngleTilt"]), 'anglePsi' : float(valueSet["_rlnAnglePsi"]), 'originX' : float(valueSet["_rlnOriginX"]), 'originY' : float(valueSet["_rlnOriginY"]), } partdict[paramdict['partnum']] = paramdict partnum = partnum + 1 if len(partdict) < 2: apDisplay.printError("No particles found in particle data file %s" % (paramfile)) apDisplay.printMsg("Processed %d particles" % (len(partdict))) return partdict
def getGoodParticles(self, classpartdatas, norefclassnum): includeParticle = [] tiltParticlesData = [] nopairParticle = 0 excludeParticle = 0 apDisplay.printMsg("sorting particles") for classpart in classpartdatas: #write to text file classnum = classpart['classNumber']-1 if classnum == norefclassnum: notstackpartnum = classpart['noref_particle']['particle']['particleNumber'] tiltstackpartdata = apTiltPair.getStackParticleTiltPair(self.params['notstackid'], notstackpartnum, self.params['tiltstackid']) if tiltstackpartdata is None: nopairParticle += 1 else: emantiltstackpartnum = tiltstackpartdata['particleNumber']-1 includeParticle.append(emantiltstackpartnum) tiltParticlesData.append(tiltstackpartdata) else: excludeParticle += 1 includeParticle.sort() apDisplay.printMsg("Keeping "+str(len(includeParticle))+" and excluding \n\t" +str(excludeParticle)+" particles with "+str(nopairParticle)+" unpaired particles") if len(includeParticle) < 1: apDisplay.printError("No particles were kept") return includeParticle, tiltParticlesData
def insertAnalysis(self, imagicstack, runtime, insert=False): ### create MSAParam object msaq = appiondata.ApImagicAlignAnalysisData() msaq['runname'] = self.params['runname'] msaq['run_seconds'] = runtime msaq['bin'] = self.params['bin'] msaq['highpass'] = self.params['hpfilt'] msaq['lowpass'] = self.params['lpfilt'] msaq['mask_radius'] = self.params['mask_radius'] msaq['mask_dropoff'] = self.params['mask_dropoff'] msaq['numiters'] = self.params['numiters'] msaq['overcorrection'] = self.params['overcorrection'] msaq['MSAdistance'] = self.params['MSAdistance'] msaq['eigenimages'] = "eigenimages" ### finish analysis run analysisrunq = appiondata.ApAlignAnalysisRunData() analysisrunq['runname'] = self.params['runname'] analysisrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) analysisrunq['imagicMSArun'] = msaq analysisrunq['alignstack'] = self.alignstackdata analysisrunq['hidden'] = False analysisrunq['description'] = self.params['description'] apDisplay.printMsg("inserting Align Analysis Run parameters into database") if insert is True: analysisrunq.insert() return
def readFile(self, oldmrcfile): apDisplay.printMsg('Reading %s into memory' % oldmrcfile) imagearray = mrc.read(oldmrcfile) # invert image density if self.params['invert'] is True: imagearray *= -1.0 return imagearray
def getTransformImageIds(transformdata): t0 = time.time() img1 = transformdata.special_getitem('image1', dereference=False).dbid img2 = transformdata.special_getitem('image2', dereference=False).dbid if time.time()-t0 > 0.3: apDisplay.printMsg("long image query "+apDisplay.timeString(time.time()-t0)) return img1, img2
def insertFSCData(self, iternum, refineIterData): fscfile = 'fsc.eotest.%d'%(iternum) fscpath = os.path.join(self.params['rundir'], "iter%03d"%(iternum), fscfile) if not os.path.isfile(fscpath): apDisplay.printWarning("Could not find FSC file: "+fscpath) return None f = open(fscpath, 'r') apDisplay.printMsg("inserting FSC Data into database") numinserts = 0 for line in f: fscq = appiondata.ApFSCData() fscq['refineIter'] = refineIterData sline = line.strip() bits = sline.split('\t') fscq['pix'] = int(bits[0]) fscq['value'] = float(bits[1]) numinserts+=1 if self.params['commit'] is True: fscq.insert() apDisplay.printMsg("inserted "+str(numinserts)+" rows of FSC data into database") f.close()
def getParticleTiltRotationAnglesOTR(stackpartdata): partdata = stackpartdata['particle'] imgnum, transformdata, otherpartdata = getTiltTransformFromParticle(partdata) t0 = time.time() tiltangle1, tiltangle2 = apDatabase.getTiltAnglesDegFromTransform(transformdata) if time.time()-t0 > 1.0: apDisplay.printMsg("long angle query "+apDisplay.timeString(time.time()-t0)) if imgnum == 1: ### negative case, tilt picker theta < 0 tiltrot = transformdata['image1_rotation'] theta = transformdata['tilt_angle'] notrot = transformdata['image2_rotation'] tiltangle = tiltangle1 - tiltangle2 elif imgnum == 2: ### positive case, tilt picker theta > 0 tiltrot = transformdata['image2_rotation'] theta = transformdata['tilt_angle'] notrot = transformdata['image1_rotation'] tiltangle = tiltangle2 - tiltangle1 else: #no particle pair info was found or some other problem print partdata apDisplay.printError("failed to get tilt pair data or some other problem") if transformdata.timestamp < datetime.datetime(2009, 2, 19, 0, 0, 0): ### bugfix for switched tilt axis angles, before Feb 19, 2009 #apDisplay.printWarning("Switching angles") temprot = notrot notrot = tiltrot tiltrot = temprot #print "tr=%.2f, th=%.2f, nr=%.2f, tilt=%.2f"%(tiltrot, theta, notrot, tiltangle) return tiltrot, theta, notrot, tiltangle
def onAdd(self, evt): vertices = [] vertices = self.panel.getTargetPositions('Region to Remove') apDisplay.printMsg("Removing region contained in %d polygon vertices"%(len(vertices))) def reversexy(coord): clist=list(coord) clist.reverse() return tuple(clist) vertices = map(reversexy,vertices) maskimg = leginon.polygon.filledPolygon(self.panel.imagedata.shape,vertices) type(maskimg) for label in self.labels: targets = self.panel.getTargets(label) eliminated = 0 newparticles = [] for target in targets: coord = (target.y,target.x) if maskimg[coord] != 0: eliminated += 1 else: newparticles.append(target) apDisplay.printMsg("%d particle(s) eliminated due to masking"%(eliminated)) self.panel.setTargets(label,newparticles) self.panel.setTargets('Region to Remove', [])
def runKerdenSOM(self, indata): """ From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/KerDenSOM KerDenSOM stands for "Kernel Probability Density Estimator Self-Organizing Map". It maps a set of high dimensional input vectors into a two-dimensional grid. """ apDisplay.printMsg("Running KerDen SOM") outstamp = os.path.join(self.params['rundir'], self.timestamp) kerdencmd = ( "xmipp_classify_kerdensom -verb 1 -i %s -o %s -xdim %d -ydim %d -saveclusters "% (indata, outstamp, self.params['xdim'], self.params['ydim']) ) ### convergence criteria if self.params['converge'] == "fast": kerdencmd += " -eps 1e-5 " elif self.params['converge'] == "slow": kerdencmd += " -eps 1e-9 " else: kerdencmd += " -eps 1e-7 " apDisplay.printColor(kerdencmd, "cyan") proc = subprocess.Popen(kerdencmd, shell=True) proc.wait() time.sleep(1) return
def start(self): aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid']) boxsize = aligndata['boxsize'] apix = aligndata['pixelsize'] maskpixrad = self.params['maskrad']/apix if maskpixrad*2 > boxsize-2: apDisplay.printError("Mask radius is too big for boxsize: %d > %d"%(maskpixrad*2,boxsize-2)) apDisplay.printMsg("Mask radius and boxsize: %.1f < %d"%(maskpixrad*2,boxsize-2)) self.instack = os.path.join(aligndata['path']['path'], aligndata['imagicfile']) outdata = "stack.data" apXmipp.convertStackToXmippData(self.instack, outdata, maskpixrad, boxsize, numpart=self.params['numpart']-1) self.runKerdenSOM(outdata) if apFile.stackSize(self.instack) > 3.0*(1024**3): # Big stacks use eman self.createMontageByEMAN() binned = None else: binned = self.createMontageInMemory(apix) self.insertKerDenSOM(binned=binned) apFile.removeFile(outdata) apFile.removeFilePattern("*.cod")
def loopProcessImage(self, imgdata): #creates self.peaktree and filterLoop sets self.filtarray self.peaktree = filterLoop.FilterLoop.loopProcessImage(self, imgdata) if self.params['background'] is False: apDisplay.printMsg("Found "+str(len(self.peaktree))+" particles for " +apDisplay.shortenImageName(imgdata['filename'])) self.stats['lastpeaks'] = len(self.peaktree) if self.params['nojpegs'] is False: if self.threadJpeg is True: threading.Thread(target=apPeaks.createPeakJpeg, args=(imgdata, self.peaktree, self.params, self.filtarray)).start() else: apPeaks.createPeakJpeg(imgdata, self.peaktree, self.params, self.filtarray) elif self.params['background'] is False: apDisplay.printWarning("Skipping JPEG creation") if self.params['defocpair'] is True: self.sibling, self.shiftpeak = apDefocalPairs.getShiftFromImage(imgdata, self.params['sessionname']) return
def createMontageInMemory(self): apDisplay.printMsg("Converting files") #logging.debug('Inside createMontageInMemory') montagepngs = [] files = glob.glob('spectra_??_??.png') for pngfile in files: montagepngs.append(pngfile) ### create montage apFile.removeFile("montage.png") montagecmd = "montage -geometry +4+4 -tile %dx%d " % ( self.params['xdim'], self.params['ydim']) for monpng in montagepngs: montagecmd += monpng + " " montagecmd += "montage.png" #logging.debug('montagecmd ' + montagecmd) proc = subprocess.Popen(montagecmd, shell=True) proc.wait() time.sleep(1)
def readRefDocFile(self): reflist = [] docfile = "ref" + self.params['timestamp'] + ".doc" if not os.path.isfile(docfile): apDisplay.printError("could not find doc file " + docfile + " to read reference angles") f = open(docfile, "r") mininplane = 360.0 for line in f: if line[:2] == ' ;': continue spidict = operations.spiderInLine(line) refdict = self.spidict2partdict(spidict) if refdict['inplane'] < mininplane: mininplane = refdict['inplane'] reflist.append(refdict) for refdict in reflist: refdict['inplane'] = refdict['inplane'] - mininplane apDisplay.printMsg("read rotation and shift parameters for " + str(len(reflist)) + " references") return reflist
def insertFSC(self, fscfile, refineIterData, commit=True): if not os.path.isfile(fscfile): apDisplay.printWarning("Could not open FSC file: " + fscfile) f = open(fscfile, 'r') apDisplay.printMsg("inserting FSC Data into database") numinserts = 0 for line in f: fscq = appiondata.ApFSCData() fscq['refineIter'] = refineIterData line = line.rstrip() bits = line.split('\t') fscq['pix'] = int(bits[0]) fscq['value'] = float(bits[1]) numinserts += 1 if commit is True: fscq.insert() apDisplay.printMsg("inserted " + str(numinserts) + " rows of FSC data into database") f.close()
def getTimestamp(self): timestamp = None if self.params['timestamp'] is not None: return self.params['timestamp'] if self.params["jobid"] is not None: jobdata = appiondata.ApSparxISACJobData.direct_query( self.params["jobid"]) timestamp = jobdata['timestamp'] elif timestamp is None: wildcard = "isac-*-params.pickle" files = glob.glob(wildcard) if len(files) == 0: apDisplay.printError("Could not determine timestamp\n" + "please provide it, e.g. -t 08nov27e54") reg = re.match("isac-([0-9a-z]*)-", files[0]) if len(reg.groups()) == 0: apDisplay.printError("Could not determine timestamp\n" + "please provide it, e.g. -t 08nov27e54") timestamp = reg.groups()[0] apDisplay.printMsg("Found timestamp = '" + timestamp + "'") return timestamp
def normalLeastSquares(X, Y): """ solve using the normal equations with no manipulation """ t0 = time.time() ### check the input if checkMatrixSizes(X, Y) is False: return None ### solve it XT = numpy.transpose(X) # create a square matrix XTX = numpy.dot(XT, X) if numpy.linalg.det(XTX) == 0: apDisplay.printWarning("Singular matrix in calculation") return None XTXinv = numpy.linalg.inv(XTX) beta = numpy.dot(numpy.dot(XTXinv, XT), Y) apDisplay.printMsg("normalLeastSquares completed in %s" % (apDisplay.timeString(time.time() - t0))) return beta
def uploadZProjection(runname, initialimagedata, uploadfile): presetdata = leginon.leginondata.PresetData( initializer=initialimagedata['preset']) presetdata['name'] = 'Zproj' imagedata = leginon.leginondata.AcquisitionImageData( initializer=initialimagedata) basename = os.path.basename(uploadfile) splitname = os.path.splitext(basename) names = splitname[0].split('_zproject') projectionname = names[0] + '_' + runname + '_zproject' imagedata['filename'] = projectionname imagedata['label'] = 'projection' imagedata['preset'] = presetdata newimgfilepath = os.path.join(imagedata['session']['image path'], projectionname) apDisplay.printMsg("Copying original image to a new location: " + newimgfilepath) shutil.copyfile(uploadfile, newimgfilepath) image = mrc.read(newimgfilepath) imagedata['image'] = image return publish(imagedata)
def start(self): self.runtime = 0 self.checkAffPropRun() self.numpart = self.getNumAlignedParticles() alignedstack = self.prepareStack() ### run Affinity Propagation aptime = time.time() classes = self.runAffinityPropagation(alignedstack) aptime = time.time() - aptime ### insert into database inserttime = time.time() self.runtime = aptime self.insertAffinityPropagationRun(classes) inserttime = time.time() - inserttime apDisplay.printMsg("Affinity propagation time: " + apDisplay.timeString(aptime)) apDisplay.printMsg("Database Insertion time: " + apDisplay.timeString(inserttime))
def convertSQLtoTree(results): t0 = time.time() parttree = [] for row in results: if len(row) < 3: apDisplay.printError("delete MySQL cache file and run again") try: if row[2] < 1.0: partpair1 = { 'part1': int(row[0]), 'part2': int(row[1]), 'tilt': False } partpair2 = { 'part1': int(row[1]), 'part2': int(row[0]), 'tilt': True } else: partpair1 = { 'part1': int(row[0]), 'part2': int(row[1]), 'tilt': True } partpair2 = { 'part1': int(row[1]), 'part2': int(row[0]), 'tilt': False } parttree.append(partpair1) parttree.append(partpair2) except: print row apDisplay.printError("bad row entry") apDisplay.printMsg("Converted "+str(len(parttree))+" particles in "+apDisplay.timeString(time.time()-t0)) return parttree
def getSymmetryDataFromName(symtext='c1', msg=True): # find the symmetry entry (full symmetry name) in the database if len(symtext.split()) > 1: symdataq = appiondata.ApSymmetryData(symmetry=symtext.strip()) else: # based on the text version from EMAN # first convert to lower case symtext = symtext.lower().strip() if symtext == "i": symtext = "icos" symdataq = appiondata.ApSymmetryData(eman_name=symtext) symdatas = symdataq.query() if not symdatas: apDisplay.printError("No symmetry named %s was found" % (symtext)) # select oldest match symdata = symdatas[len(symdatas) - 1] if msg is True: apDisplay.printMsg("Selected symmetry group: " + apDisplay.colorString( "%s -- %s" % (symdata['eman_name'].upper(), symdata['symmetry']), "cyan")) return symdata
def stackHDFToIMAGIC(hdfFile, imagicFile=None): """ convert HDF stack back into an IMAGIC stack """ numPart = EMAN2.EMUtil.get_image_count(hdfFile) # output must end with hdf root, ext = os.path.splitext(hdfFile) if imagicFile is None or ext != ".hed": imagicFile = root + ".hed" apFile.removeFile(imagicFile) apFile.removeFile(root + ".img") apDisplay.printMsg("Creating IMAGIC file '%s' with %d particles" % (imagicFile, numPart)) headerOnly = False for i in range(numPart): imgData = EMAN2.EMData.read_images(hdfFile, [i], headerOnly)[0] imgData.write_image(imagicFile, i) return imagicFile
def runSatAverage(self): keepfile = os.path.join(self.params['rundir'], "keeplist-tot"+self.datastr+".lst") newname = "recon%d_cut%d_iter%d.hed" % (self.params['reconid'], self.params['cutrange']*10, self.iternum) volname = "recon%d_cut%d_iter%d.%da.mrc"% (self.params['reconid'], self.params['cutrange']*10, self.iternum, self.iternum) volfile = os.path.join(self.params['rundir'], "volumes", volname) if os.path.isfile(volfile): apDisplay.printMsg("Skipping sat average, volume exists") return cmd = ( "satAverage.py " +" --projectid="+str(self.params['projectid']) +" --reconid="+str(self.params['reconid']) +" \\\n --mask=40 --iter="+str(self.iternum) +" \\\n --stackname="+newname +" \\\n --keep-list="+keepfile +" \n" ) print "New satAverage.py Command:" apDisplay.printColor(cmd, "purple") if self.params['sataverage'] is True: proc = subprocess.Popen(cmd, shell=True) proc.communicate() return
def getImageIdsFromStack(stackid, msg=True): if stackid < 1: return [] t0 = time.time() stackdata = appiondata.ApStackData.direct_query(stackid) stackq = appiondata.ApStackParticleData() stackq['stack'] = stackdata stackparticledata = stackq.query() stackimages = [] if msg is True: apDisplay.printMsg("querying particle images from stackid=" + str(stackid) + " on " + time.asctime()) for sp in stackparticledata: spimagedata = sp['particle']['image'] spimageid = spimagedata.dbid if spimageid not in stackimages: stackimages.append(spimageid) if msg is True: apDisplay.printMsg("Found %d images from stackid=%d" % (len(stackimages), stackid)) return stackimages
def checkStackNumbering(stackname): import EMAN # check that the numbering is stored in the NImg parameter apDisplay.printMsg("checking that original stack is numbered") n = EMAN.fileCount(stackname)[0] im = EMAN.EMData() im.readImage(stackname, n - 1) # if last particle is not numbered with same value as # of particles, # renumber the entire stack if n - 1 != im.NImg(): apDisplay.printWarning( "Original stack is not numbered! numbering now...") # check that the stack exists in a writable directory and is not read-only if os.access(stackname, os.W_OK) is True: numberParticlesInStack(stackname, startnum=0, verbose=True) else: apDisplay.printWarning( "old stack header exists in a READ-only directory and cannot be numbered according to EMAN" ) return
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['description'] is None: apDisplay.printError("run description was not defined") if self.params['templatelist'] is None: apDisplay.printError("template list was not provided") if self.params['lastring'] is None: apDisplay.printError("a last ring radius was not provided") if self.params['runname'] is None: apDisplay.printError("run name was not defined") stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize( self.params['stackid']) / self.params['bin'] if self.params['lastring'] > boxsize / 2 - 2: apDisplay.printError("last ring radius is too big for boxsize " + str(self.params['lastring']) + " > " + str(boxsize / 2 - 2)) if self.params['lastring'] + self.params['xysearch'] > boxsize / 2 - 2: apDisplay.printError( "last ring plus xysearch radius is too big for boxsize " + str(self.params['lastring'] + self.params['xysearch']) + " > " + str(boxsize / 2 - 2)) ### convert / check template data self.templatelist = self.params['templatelist'].strip().split(",") if not self.templatelist or type(self.templatelist) != type([]): apDisplay.printError("could not parse template list=" + self.params['templatelist']) self.params['numtemplate'] = len(self.templatelist) apDisplay.printMsg("Found " + str(self.params['numtemplate']) + " templates")
def alignPicks2(picks1, picks2, data, limit=20.0, msg=True): ### create distance dictionary alignpicks2 = a2Toa1Data(picks2, data) sortedDict2 = {} index = 0 while index < len(alignpicks2): p2 = alignpicks2[index] key = "%d,%d" % ( p2[0] / limit, p2[1] / limit, ) if not key in sortedDict2: sortedDict2[key] = [ index, ] else: sortedDict2[key].append(index) index += 1 ### find matching picks filled = {} list1 = [] alignlist2 = [] for p1 in picks1: closepick, dist = findClosestPick2(p1, alignpicks2, sortedDict2, limit) if dist < limit: key = str(closepick) if not key in filled: list1.append(p1) alignlist2.append(closepick) filled[key] = True #convert lists nlist1 = numpy.array(list1, dtype=numpy.int32) nalignlist2 = numpy.array(alignlist2, dtype=numpy.int32) #transform back nlist2 = a1Toa2Data(nalignlist2, data) if msg is True: apDisplay.printMsg("Aligned "+str(len(nlist1))+" of "+str(len(picks1))+\ " particles to "+str(len(nlist2))+" of "+str(len(picks2))) return nlist1, nlist2
def sortFolder(self): numsort = 0 apDisplay.printMsg("Sorting files into clean folders") ### move files for all particle iterations files = [] for i in range(self.lastiter+1): iterdir = "iter%03d"%(i) apParam.createDirectory(iterdir, warning=False) wildcard = "part*_it*%03d_*.*"%(i) files.extend(glob.glob(wildcard)) wildcard = "part*_it*%03d.*"%(i) files.extend(glob.glob(wildcard)) for filename in files: if os.path.isfile(filename): numsort += 1 shutil.move(filename,iterdir) if numsort < 3: apDisplay.printWarning("Problem in iteration file sorting, are they already sorted?") apDisplay.printMsg("Sorted "+str(numsort)+" iteration files") ### move files for all reference iterations refsort = 0 refdir = "refalign" apParam.createDirectory(refdir, warning=False) wildcard = "ref*_it*.*" files = glob.glob(wildcard) for filename in files: refsort += 1 shutil.move(filename, refdir) #if refsort < 5: # apDisplay.printError("Problem in reference file sorting") apDisplay.printMsg("Sorted "+str(refsort)+" reference files") return
def _removeProcessedImages(self): startlen = len(self.imgtree) donecount = 0 reproccount = 0 rejectcount = 0 self.stats['skipcount'] = 0 newimgtree = [] count = 0 t0 = time.time() for imgdata in self.imgtree: count += 1 if count % 10 == 0: sys.stderr.write(".") skip, reason = self.skipTestOnImage(imgdata) imgname = imgdata['filename'] if skip is True: if reason == 'reproc': reproccount += 1 if skip is True: if self.stats['skipcount'] == 0: sys.stderr.write("skipping processed images\n") elif self.stats['skipcount'] % 80 == 0: sys.stderr.write(".\n") else: sys.stderr.write(".") self.stats['skipcount'] += 1 else: newimgtree.append(imgdata) sys.stderr.write("\n") apDisplay.printMsg("finished skipping in %s" % (apDisplay.timeString(time.time() - t0))) if self.stats['skipcount'] > 0: self.imgtree = newimgtree sys.stderr.write("\n") apDisplay.printWarning("skipped " + str(self.stats['skipcount']) + " of " + str(startlen) + " images") apDisplay.printMsg("[[ " + str(reproccount) + " no reprocess " + " | " + str(rejectcount) + " rejected " + " | " + str(donecount) + " in donedict ]]")
def runMakeSpectra(self, indata): """ From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Makespectra This program generates a Fourier-Bessel decomposition of each image listed in a *.sel. Then creates a report ASCII file for each image, with the harmonic energy percentage as a function of the radius, with extension .SPT. """ spectratime = time.time() apDisplay.printMsg("Running make_spectra") #logging.debug('Inside make_spectra') #ROB tempFileNameforSpectra = "tempFileNameforSpectra.txt" tmpSelFile = apXmipp.breakupStackIntoSingleFiles( indata, numpart=self.params['numpart']) makespectracmd = ("xmipp_make_spectra -i %s -o %s \ -x0 %f -y0 %f -r1 %d -r2 %d -low %d -high %d " % ( tmpSelFile, tempFileNameforSpectra, self.xOffset, self.yOffset, self.params['spectrainnerradius'], self.params['spectraouterradius'], self.params['spectralowharmonic'], self.params['spectrahighharmonic'], )) #logging.debug(makespectracmd) apDisplay.printColor(makespectracmd, "cyan") proc = subprocess.Popen(makespectracmd, shell=True, stdout=subprocess.PIPE) proc.wait() apDisplay.printMsg("Spectra calculation complete in %s" % (apDisplay.timeString(time.time() - spectratime))) time.sleep(1) return tempFileNameforSpectra
def start(self): """ this is the main component of the script where all the processing is done """ ### initialize some variables self.runq = None self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) apDisplay.printMsg("Pixel size: %.5f"%(self.apix)) self.boxsize = apStack.getStackBoxsize(self.params['stackid']) apDisplay.printMsg("Box size: %d"%(self.boxsize)) self.checkResults() self.stackmapping = apRecon.partnum2defid(self.params['stackid']) self.numiter = self.getNumberOfIterations() for i in range(self.numiter): iternum = i+1 apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green") self.uploadIteration(iternum) reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir']) if reconrunid: apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid']) apRecon.setGoodBadParticlesFromReconId(reconrunid) else: apDisplay.printWarning("Could not find recon run id")
def imageToArray(im, convertType='uint8', dtype=None, msg=True): """ Convert PIL image to numpy array copied and modified from http://mail.python.org/pipermail/image-sig/2005-September/003554.html """ if im.mode == "L": a = numpy.fromstring(im.tostring(), numpy.uint8) a = numpy.reshape(a, (im.size[1], im.size[0])) #a.shape = (im.size[1], im.size[0], 1) # alternate way elif (im.mode == 'RGB'): apDisplay.printMsg("reading RGB and converting to L") grey = im.convert('L') a = numpy.fromstring(grey.tostring(), numpy.uint8) a = numpy.reshape(a, (grey.size[1], grey.size[0])) elif (im.mode == 'RGBA'): apDisplay.printMsg("reading RGBA and converting to L") grey = im.convert('L') a = numpy.fromstring(grey.tostring(), numpy.uint8) a = numpy.reshape(a, (grey.size[1], grey.size[0])) elif (im.mode == 'LA'): apDisplay.printMsg("reading LA and converting to L") grey = im.convert('L') a = numpy.fromstring(grey.tostring(), numpy.uint8) a = numpy.reshape(a, (grey.size[1], grey.size[0])) else: raise ValueError, im.mode + " mode not considered" if convertType == 'float32': a = a.astype(numpy.float32) if dtype is not None: a = a.astype(dtype) return a
def importPicks(self, picks1, picks2, tight=False, msg=True): t0 = time.time() #print picks1 #print self.currentpicks1 curpicks1 = numpy.asarray(self.currentpicks1) curpicks2 = numpy.asarray(self.currentpicks2) #print curpicks1 # get picks apTiltTransform.setPointsFromArrays(curpicks1, curpicks2, self.data) pixdiam = self.data['pixdiam'] if tight is True: pixdiam /= 4.0 #print self.data, pixdiam list1, list2 = apTiltTransform.alignPicks2(picks1, picks2, self.data, limit=pixdiam, msg=msg) if list1.shape[0] == 0 or list2.shape[0] == 0: apDisplay.printWarning("No new picks were found") # merge picks newpicks1, newpicks2 = apTiltTransform.betterMergePicks(curpicks1, list1, curpicks2, list2, msg=msg) newparts = newpicks1.shape[0] - curpicks1.shape[0] # copy over picks self.currentpicks1 = newpicks1 self.currentpicks2 = newpicks2 if msg is True: apDisplay.printMsg("Inserted " + str(newparts) + " new particles in " + apDisplay.timeString(time.time() - t0)) return True
def getPartcileLists(self): #first query query1 = self.queryParticles(swap=False) self.cursor.execute(query1) results1 = self.cursor.fetchall() apDisplay.printMsg("Found "+str(len(results1))+" particle pairs in forward order") #if len(results1) < 2: # apDisplay.printError("Failed to find any particles") #swap particle1 and particle2 in ApTiltParticlePairData query2 = self.queryParticles(swap=True) self.cursor.execute(query2) results2 = self.cursor.fetchall() apDisplay.printMsg("Found "+str(len(results2))+" particle pairs in reverse order") #if len(results2) < 2: # apDisplay.printError("Failed to find any particles") parttree = self.parseResults(results1, results2) f = open("tiltsync-"+self.timestamp+".dat", "w") count = 0 for partdict in parttree: count += 1 emannotnum = partdict['not']-1 emantiltnum = partdict['tilt']-1 line = operations.spiderOutLine(count, (emannotnum,emantiltnum)) f.write(line) f.close() apDisplay.printMsg("Writing "+str(len(parttree))+" particle pairs") if len(parttree) < 2: apDisplay.printError("Failed to find any particle pairs") return parttree
def getResolutionFromFSCFile(fscfile, boxsize, apix, msg=False): """ should use more general apFourier.getResolution() """ if not os.path.isfile(fscfile): apDisplay.printError("fsc file does not exist") if msg is True: apDisplay.printMsg("box: %d, apix: %.3f, file: %s" % (boxsize, apix, fscfile)) f = open(fscfile, 'r') lastx = 0 lasty = 0 for line in f: xy = line.strip().split() x = float(xy[0]) y = float(xy[1]) if x != 0.0 and x < 0.9: apDisplay.printWarning("FSC is wrong data format") if y > 0.5: #store values for later lastx = x lasty = y else: # get difference of fsc diffy = lasty - y # get distance from 0.5 distfsc = (0.5 - y) / diffy # get interpolated spatial freq intfsc = x - distfsc * (x - lastx) # convert to Angstroms if intfsc > 0.0: res = boxsize * apix / intfsc else: res = boxsize * apix f.close() return res # fsc did not fall below 0.5 apDisplay.printWarning("Failed to determine resolution") res = boxsize * apix / (lastx + 1) return res
def optimizeAngles(self, msg=True): t0 = time.time() ### run find theta na1 = numpy.array(self.currentpicks1, dtype=numpy.int32) na2 = numpy.array(self.currentpicks2, dtype=numpy.int32) # minimum area for a triangle to be valid arealim = 100.0 fittheta = radermacher.tiltang(na1, na2, arealim) if not fittheta or not 'wtheta' in fittheta: return theta = fittheta['wtheta'] thetadev = fittheta['wthetadev'] if msg is True: thetastr = ("%3.3f +/- %2.2f" % (theta, thetadev)) tristr = apDisplay.orderOfMag( fittheta['numtri']) + " of " + apDisplay.orderOfMag( fittheta['tottri']) tristr = (" (%3.1f " % (100.0 * fittheta['numtri'] / float(fittheta['tottri']))) + "%) " apDisplay.printMsg("Tilt angle " + thetastr + tristr) self.data['theta'] = fittheta['wtheta'] ### run optimize angles lastiter = [80, 80, 80] count = 0 totaliter = 0 while max(lastiter) > 75 and count < 30: count += 1 lsfit = self.runLeastSquares() lastiter[2] = lastiter[1] lastiter[1] = lastiter[0] lastiter[0] = lsfit['iter'] totaliter += lsfit['iter'] if msg is True: apDisplay.printMsg("Least Squares: " + str(count) + " rounds, " + str(totaliter) + " iters, rmsd of " + str(round(lsfit['rmsd'], 4)) + " pixels in " + apDisplay.timeString(time.time() - t0)) return
def start(self): parttree = apFrealign.parseFrealignParamFile(self.params['paramfile']) phases = [] for partdict in parttree: phases.append(partdict['phase_residual']) del parttree phases.sort() phases = numpy.array(phases, dtype=numpy.float32) if self.params['gracex'] is None: apDisplay.printMsg("xFile: %s" % (self.params['paramfile'])) apDisplay.printMsg("xMean: %.3f +/- %.3f (%d total)" % (phases.mean(), phases.std(), phases.shape[0])) apDisplay.printMsg( "xRange: %.2f (min) <> %.2f (0.1) <> %.2f (0.25) <> %.2f (med) <> %.2f (0.75) <> %.2f (0.9) <> %.2f (max)" % (phases.min(), phases[int(phases.shape[0] * 0.1)], phases[int(phases.shape[0] * 0.25)], numpy.median(phases), phases[int(phases.shape[0] * 0.75)], phases[int( phases.shape[0] * 0.9)], phases.max())) else: f = open("boxplot.dat", "a") #boxplot line: (X, median, upper/lower limit, upper/lower whisker) f.write("%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n" % ( self.params['gracex'], numpy.median(phases), phases[int(phases.shape[0] * 0.75)], phases[int(phases.shape[0] * 0.25)], phases[int(phases.shape[0] * 0.9)], phases[int(phases.shape[0] * 0.1)], )) f.close() f = open("means.dat", "a") f.write("%d\t%.8f\n" % (self.params['gracex'], phases.mean())) f.close()
def loadOneRawFrame(self, rawframe_path, frame_number): ''' Load one raw frame depending on the rawframetype ''' if self.getRawFrameType() == 'singles': return super(GatanK2Processing, self).loadOneRawFrame(rawframe_path, frame_number) ''' Load from rawframe_path (a stack file) the chosen frame of the current image. ''' try: bin = self.camerainfo['binning'] offset = self.camerainfo['offset'] dimension = self.camerainfo['dimension'] except: # default bin = {'x': 1, 'y': 1} offset = {'x': 0, 'y': 0} dimension = self.getDefaultDimension() crop_end = { 'x': offset['x'] + dimension['x'] * bin['x'], 'y': offset['y'] + dimension['y'] * bin['y'] } apDisplay.printMsg('Frame path: %s' % rawframe_path) waitmin = 0 while not os.path.exists(rawframe_path): if self.waittime < 0.1: apDisplay.printWarning('Frame File %s does not exist.' % rawframe_path) return False apDisplay.printWarning( 'Frame File %s does not exist. Wait for 3 min.' % rawframe_path) time.sleep(180) waitmin += 3 apDisplay.printMsg('Waited for %d min so far' % waitmin) if waitmin > self.waittime: return False return self.readImageFrame(rawframe_path, frame_number, offset, crop_end, bin)
def insertImgAssessmentStatus(imgdata, runname="run1", assessment=None, msg=True): """ Insert the assessment status keep = True reject = False unassessed = None """ if assessment is True or assessment is False: assessrun = appiondata.ApAssessmentRunData() assessrun['session'] = imgdata['session'] #override to ALWAYS be 'run1' #assessrun['name'] = runname assessrun['name'] = "run1" assessquery = appiondata.ApAssessmentData() assessquery['image'] = imgdata assessquery['assessmentrun'] = assessrun assessquery['selectionkeep'] = assessment assessquery.insert() else: apDisplay.printWarning("No image assessment made, invalid data: " + str(assessment)) #check assessment if msg is True: finalassess = getImgAssessmentStatus(imgdata) imgname = apDisplay.short(imgdata['filename']) if finalassess is True: astr = apDisplay.colorString("keep", "green") elif finalassess is False: astr = apDisplay.colorString("reject", "red") elif finalassess is None: astr = apDisplay.colorString("none", "yellow") apDisplay.printMsg("Final image assessment: " + astr + " (" + imgname + ")") return True
def commitToJensenDatabase(session_time, fulltomodata, stackdir, processdir, stackname, description): if not upload_to_Jensen_database: return raptordatabase = 0 apDisplay.printMsg("Uploding to Jensen tomography database") alignrun = fulltomodata['alignrun'] tiltseries = fulltomodata['tiltseries'] nn = len(stackname) mrcbase = processdir + '/align/' + stackname[0:-3] mrcpath = mrcbase + '_full.rec' if not os.path.exists(mrcpath): mrcpath = mrcbase + '_part.rec' if not os.path.exists(mrcpath): mrcpath = '' if mrcpath == '': apDisplay.printWarning('No output full or part mrc found.') raptordatabase = 1 else: print '!!!0', ' mrcpath=', mrcpath print '!!!1', ' stackdir=', stackdir, ' stackname=', stackname, ' processdir=', processdir, ' description=', description imagelist = apTomo.getImageList([tiltseries]) defocus = imagelist[0]['preset']['defocus'] magnification = imagelist[0]['preset']['magnification'] q = appiondata.ApTiltsInAlignRunData(alignrun=alignrun) r = q.query() tomosettings = r[0]['settings'] tilt_min = tomosettings['tilt min'] tilt_max = tomosettings['tilt max'] tilt_step = tomosettings['tilt step'] dose = tomosettings['dose'] print '!!!2', ' defocus=', defocus, ' magnification=', magnification, ' tilt_min=', tilt_min, ' tilt_max=', tilt_max, ' tilt_step=', tilt_step, ' dose=', dose #sessiondata = tiltdatalist[0]['session'] session_id = tiltseries.dbid print '!!!3', ' session_id=', session_id, ' session_time=', session_time return raptordatabase
def calculateEulerJumpsAndGoodBadParticles(self, uploadIterations): ''' calculate euler jumps for entire recon, currently based on EMAN ZXZ convention ''' if self.params['commit'] is True: reconrunid = self.refinerunq.dbid ### make table entries of good-bad particles apRecon.setGoodBadParticlesFromReconId(reconrunid) ### verify completed refinements and iterations refinecomplete = self.verifyNumberOfCompletedRefinements( multiModelRefinementRun=self.multiModelRefinementRun) if self.multiModelRefinementRun is True: ### Euler jumpers calculated from ApMultiModelRefineRunData in multi-model case, looping over values in all single-model refinements multimodelrunid = self.multimodelq.dbid if len(uploadIterations) > 1: apDisplay.printMsg( "calculating euler jumpers for multi-model refinement=" + str(multimodelrunid)) eulerjump = apEulerJump.ApEulerJump() ### TECHNICALLY, IT DOESN'T MAKE SENSE TO PASS THE RECONRUNID, SINCE PARTICLES CAN BE JUMPING, ### BUT AS FAR AS I CAN TELL, IT DOESN'T MAKE A DIFFERENCE IN THE RESULTING QUERY, SINCE THE VARIABLE ### IS ONLY USED TO QUERY FOR STACK PARTICLES, WHICH ARE IDENTICAL IN MULTI-MODEL AND SINGLE-MODEL REFINEMENT ### CASES. THEREFORE, THE LAST RECONRUNID IS PASSES ... * DMITRY eulerjump.calculateEulerJumpsForEntireRecon( reconrunid, self.runparams['stackid'], multimodelrunid=multimodelrunid) else: ### Euler jumpers calculated from ApRefineRunData in single-model case if len(uploadIterations) > 1 or \ (refinecomplete.itervalues().next()[-1] == self.runparams['numiter'] and len(refinecomplete.itervalues().next())>1): apDisplay.printMsg("calculating euler jumpers for recon=" + str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon( reconrunid, self.runparams['stackid']) return