def insertRefineParticleData(self, iterdata, parttree):
apDisplay.printMsg("Inserting particle data")
phase_threshold = float(iterdata['frealignParams']['thresh'])
count = 0
for partdict in parttree:
count += 1
if count % 1000 == 0:
apDisplay.printMsg("Inserted %d particles"%(count))
partrefineq = appiondata.ApRefineParticleData()
partrefineq['refineIter'] = iterdata
partnum = partdict['partnum']
stackpartid = self.stackmapping[partnum]
stackpartdata = appiondata.ApStackParticleData.direct_query(stackpartid)
partrefineq['particle'] = stackpartdata
partkeys = ('shiftx', 'shifty', 'euler1', 'euler2', 'euler3', 'phase_residual')
for key in partkeys:
partrefineq[key] = partdict[key]
### check if particle was rejected
if partdict['phase_residual'] > iterdata['frealignParams']['thresh']:
partrefineq['refine_keep'] = False
else:
partrefineq['refine_keep'] = True
partrefineq['mirror'] = False
partrefineq['postRefine_keep'] = False
partrefineq['euler_convention'] = 'zyz'
if self.params['commit'] is True:
partrefineq.insert()
return
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['description'] is None:
apDisplay.printError("run description was not defined")
if self.params['templatelist'] is None:
apDisplay.printError("template list was not provided")
if self.params['lastring'] is None:
apDisplay.printError("a last ring radius was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])/self.params['bin']
if self.params['lastring'] > boxsize/2-2:
apDisplay.printError("last ring radius is too big for boxsize "
+str(self.params['lastring'])+" > "+str(boxsize/2-2))
if self.params['lastring']+self.params['xysearch'] > boxsize/2-2:
apDisplay.printError("last ring plus xysearch radius is too big for boxsize "
+str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2))
### convert / check template data
self.templatelist = self.params['templatelist'].strip().split(",")
if not self.templatelist or type(self.templatelist) != type([]):
apDisplay.printError("could not parse template list="+self.params['templatelist'])
self.params['numtemplate'] = len(self.templatelist)
apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates")
def boxerRotate(imgfile, parttree, outstack, boxsize):
"""
boxes the particles with expanded size,
applies a rotation to particle,
reduces boxsize to requested size,
and saves them to a imagic file
"""
# size needed is sqrt(2)*boxsize, using 1.5 to be extra safe
bigboxsize = int(math.ceil(1.5*boxsize))
imgarray = mrc.read(imgfile)
bigboxedparticles = boxerMemory(imgarray, parttree, bigboxsize)
boxedparticles = []
boxshape = (boxsize,boxsize)
apDisplay.printMsg("Rotating particles...")
for i in range(len(bigboxedparticles)):
if i % 10 == 0:
sys.stderr.write(".")
bigboxpart = bigboxedparticles[i]
partdict = parttree[i]
### add 90 degrees because database angle is from x-axis not y-axis
angle = partdict['angle']+90.0
rotatepart = ndimage.rotate(bigboxpart, angle=angle, reshape=False, order=1)
boxpart = imagefilter.frame_cut(rotatepart, boxshape)
boxedparticles.append(boxpart)
sys.stderr.write("done\n")
apImagicFile.writeImagic(boxedparticles, outstack)
return True
def phaseFlipWholeImage(self, inimgpath, imgdata):
outimgpath = os.path.join(self.params['rundir'], self.shortname+"-ctfcorrect.dwn.mrc")
### High tension on CM is given in kv instead of v so do not divide by 1000 in that case
if imgdata['scope']['tem']['name'] == "CM":
voltage = imgdata['scope']['high tension']
else:
voltage = (imgdata['scope']['high tension'])/1000
apix = apDatabase.getPixelSize(imgdata)
defocus, ampconst = self.getDefocusAmpConstForImage(imgdata, True)
defocus *= 1.0e6
self.checkDefocus(defocus, self.shortname)
### get all CTF parameters, we also need to get the CS value from the database
ctfdata = self.getBestCtfValue(imgdata, False)
#ampconst = ctfdata['amplitude_contrast'] ### we could use this too
# find cs
cs = self.getCS(ctfdata)
parmstr = ("parm=%f,200,1,%.3f,0,17.4,9,1.53,%i,%.1f,%f" %(defocus, ampconst, voltage, cs, apix))
emancmd = ("applyctf %s %s %s setparm flipphase" % (inimgpath, outimgpath, parmstr))
apDisplay.printMsg("phaseflipping entire micrograph with defocus "+str(round(defocus,3))+" microns")
apEMAN.executeEmanCmd(emancmd, showcmd = True)
return outimgpath
def processParticles(self, imgdata, partdatas, shiftdata):
self.shortname = apDisplay.short(imgdata['filename'])
### if only selected points along helix,
### fill in points with helical step
if self.params['helicalstep']:
apix = apDatabase.getPixelSize(imgdata)
partdatas = self.fillWithHelicalStep(partdatas, apix)
### run batchboxer
self.boxedpartdatas, self.imgstackfile, self.partmeantree = self.boxParticlesFromImage(imgdata, partdatas, shiftdata)
if self.boxedpartdatas is None:
self.stats['lastpeaks'] = 0
apDisplay.printWarning("no particles were boxed from "+self.shortname+"\n")
self.badprocess = True
return None
self.stats['lastpeaks'] = len(self.boxedpartdatas)
apDisplay.printMsg("do not break function now otherwise it will corrupt stack")
#time.sleep(1.0)
### merge image particles into big stack
totalpart = self.mergeImageStackIntoBigStack(self.imgstackfile, imgdata)
### create a stack average every so often
if self.stats['lastpeaks'] > 0:
totalPartices = self.existingParticleNumber+self.stats['peaksum']+self.stats['lastpeaks']
logpeaks = math.log(totalPartices)
if logpeaks > self.logpeaks:
self.logpeaks = math.ceil(logpeaks)
numpeaks = math.ceil(math.exp(self.logpeaks))
apDisplay.printMsg("writing averaging stack, next average at %d particles"%(numpeaks))
mrc.write(self.summedParticles/float(totalPartices), "average.mrc")
return totalpart
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def boxParticlesFromImage(self, imgdata, partdatas, shiftdata):
### convert database particle data to coordinates and write boxfile
boxfile = os.path.join(self.params['rundir'], imgdata['filename']+".box")
parttree, boxedpartdatas = apBoxer.processParticleData(imgdata, self.boxsize,
partdatas, shiftdata, boxfile, rotate=self.params['rotate'])
if self.params['boxfiles']:
### quit and return, boxfile created, now process next image
return None, None, None
### check if we have particles again
if len(partdatas) == 0 or len(parttree) == 0:
apDisplay.printColor(self.shortname+" has no remaining particles and has been rejected\n","cyan")
return None, None, None
### set up output file path
imgstackfile = os.path.join(self.params['rundir'], self.shortname+".hed")
if (self.is_dd_stack and (self.params['nframe'] or self.params['driftlimit'])
and not self.params['phaseflipped'] and not self.params['rotate']):
# If processing on whole image is not needed, it is more efficient to use mmap to box frame stack
apDisplay.printMsg("boxing "+str(len(parttree))+" particles into temp file: "+imgstackfile)
framelist = self.dd.getFrameList(self.params)
apBoxer.boxerFrameStack(self.dd.framestackpath, parttree, imgstackfile, self.boxsize, framelist)
else:
self._boxParticlesFromImage(imgdata, parttree, imgstackfile)
partmeantree = self.calculateParticleStackStats(imgstackfile, boxedpartdatas)
imgstackfile = self.postProcessParticleStack(imgdata, imgstackfile, boxedpartdatas, len(parttree))
return boxedpartdatas, imgstackfile, partmeantree
def importPicks(self, picks1, picks2, tight=False, msg=True):
t0 = time.time()
#print picks1
#print self.currentpicks1
curpicks1 = numpy.asarray(self.currentpicks1)
curpicks2 = numpy.asarray(self.currentpicks2)
#print curpicks1
# get picks
apTiltTransform.setPointsFromArrays(curpicks1, curpicks2, self.data)
pixdiam = self.data['pixdiam']
if tight is True:
pixdiam /= 4.0
#print self.data, pixdiam
list1, list2 = apTiltTransform.alignPicks2(picks1, picks2, self.data, limit=pixdiam, msg=msg)
if list1.shape[0] == 0 or list2.shape[0] == 0:
apDisplay.printWarning("No new picks were found")
# merge picks
newpicks1, newpicks2 = apTiltTransform.betterMergePicks(curpicks1, list1, curpicks2, list2, msg=msg)
newparts = newpicks1.shape[0] - curpicks1.shape[0]
# copy over picks
self.currentpicks1 = newpicks1
self.currentpicks2 = newpicks2
if msg is True:
apDisplay.printMsg("Inserted "+str(newparts)+" new particles in "+apDisplay.timeString(time.time()-t0))
return True
def optimizeAngles(self, msg=True):
t0 = time.time()
### run find theta
na1 = numpy.array(self.currentpicks1, dtype=numpy.int32)
na2 = numpy.array(self.currentpicks2, dtype=numpy.int32)
fittheta = radermacher.tiltang(na1, na2)
if not fittheta or not 'wtheta' in fittheta:
return
theta = fittheta['wtheta']
thetadev = fittheta['wthetadev']
if msg is True:
thetastr = ("%3.3f +/- %2.2f" % (theta, thetadev))
tristr = apDisplay.orderOfMag(fittheta['numtri'])+" of "+apDisplay.orderOfMag(fittheta['tottri'])
tristr = (" (%3.1f " % (100.0 * fittheta['numtri'] / float(fittheta['tottri'])))+"%) "
apDisplay.printMsg("Tilt angle "+thetastr+tristr)
self.data['theta'] = fittheta['wtheta']
### run optimize angles
lastiter = [80,80,80]
count = 0
totaliter = 0
while max(lastiter) > 75 and count < 30:
count += 1
lsfit = self.runLeastSquares()
lastiter[2] = lastiter[1]
lastiter[1] = lastiter[0]
lastiter[0] = lsfit['iter']
totaliter += lsfit['iter']
if msg is True:
apDisplay.printMsg("Least Squares: "+str(count)+" rounds, "+str(totaliter)
+" iters, rmsd of "+str(round(lsfit['rmsd'],4))+" pixels in "+apDisplay.timeString(time.time()-t0))
return
def commitToDatabase(self, imgdata):
"""
Uses the appionLoop commit
"""
imgassess = apDatabase.getImgCompleteStatus(imgdata)
tiltdata = apTiltPair.getTiltPair(imgdata)
msg = not self.params['background']
if tiltdata is None:
if imgassess is not False:
apDisplay.printColor("\nrejecting unpaired image: "+apDisplay.short(imgdata['filename']), "red")
apDatabase.insertImgAssessmentStatus(imgdata, self.params['runid'], False, msg=msg)
self.reject+=1
return
if self.params['background'] is False:
apDisplay.printMsg("tiltpair: "+apDisplay.short(tiltdata['filename']))
tiltassess = apDatabase.getImgCompleteStatus(tiltdata)
if imgassess is False or tiltassess is False:
if imgassess is not False:
apDisplay.printColor("\nrejecting bad tilt images: "+apDisplay.short(imgdata['filename']), "magenta")
if tiltassess is not False:
apDisplay.printColor("\nrejecting bad tilt images: "+apDisplay.short(tiltdata['filename']), "magenta")
apDatabase.insertImgAssessmentStatus(imgdata, self.params['runid'], False, msg=msg)
apDatabase.insertImgAssessmentStatus(tiltdata, self.params['runid'], False, msg=msg)
self.reject+=2
if self.params['background'] is False:
print "Assessment:", imgassess, tiltassess
return
def getOverlap(self, image1, image2, msg=True):
t0 = time.time()
bestOverlap, tiltOverlap = apTiltTransform.getOverlapPercent(image1, image2, self.data)
overlapStr = str(round(100*bestOverlap,2))+"% and "+str(round(100*tiltOverlap,2))+"%"
if msg is True:
apDisplay.printMsg("Found overlaps of "+overlapStr+" in "+apDisplay.timeString(time.time()-t0))
self.data['overlap'] = bestOverlap
def convertStackToSpider(self, emanstackfile, classnum):
"""
takes the stack file and creates a spider file ready for processing
"""
if not os.path.isfile(emanstackfile):
apDisplay.printError("stackfile does not exist: "+emanstackfile)
### first high pass filter particles
apDisplay.printMsg("pre-filtering particles")
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
emancmd = ("proc2d "+emanstackfile+" "+emanstackfile
+" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
+" inplace")
apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc2d "
emancmd += emanstackfile+" "
spiderstack = os.path.join(self.params['rundir'], str(classnum), "otrstack"+self.timestamp+".spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
return spiderstack
def convertSQLtoEulerTree(self, results):
t0 = time.time()
eulertree = []
for row in results:
if len(row) < 11:
apDisplay.printError("delete MySQL cache file and run again")
try:
eulerpair = { 'part1': {}, 'part2': {} }
eulerpair['part1']['partid'] = int(row[0])
eulerpair['part1']['dbid'] = int(row[1])
eulerpair['part1']['euler1'] = float(row[2])
eulerpair['part1']['euler2'] = float(row[3])
eulerpair['part1']['euler3'] = float(row[4])
eulerpair['part1']['mirror'] = self.nullOrValue(row[5])
eulerpair['part1']['reject'] = not self.nullOrValue(row[6])
eulerpair['part1']['tilt'] = apStack.getStackParticleTilt(eulerpair['part1']['dbid'])
eulerpair['part2']['partid'] = int(row[7])
eulerpair['part2']['dbid'] = int(row[8])
eulerpair['part2']['euler1'] = float(row[9])
eulerpair['part2']['euler2'] = float(row[10])
eulerpair['part2']['euler3'] = float(row[11])
eulerpair['part2']['mirror'] = self.nullOrValue(row[12])
eulerpair['part2']['reject'] = not self.nullOrValue(row[13])
eulerpair['part2']['tilt'] = apStack.getStackParticleTilt(eulerpair['part2']['dbid'])
eulertree.append(eulerpair)
except:
print row
apDisplay.printError("bad row entry")
apDisplay.printMsg("Converted "+str(len(eulertree))+" eulers in "+apDisplay.timeString(time.time()-t0))
return eulertree
def xmippNormStack(self, inStackPath, outStackPath):
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath)
### setup Xmipp command
xmippexe = apParam.getExecPath("xmipp_normalize", die=True)
apDisplay.printMsg("Using Xmipp to normalize particle stack")
normtime = time.time()
xmippopts = ( " "
+" -i %s"%os.path.join(self.params['rundir'],selfile)
+" -method Ramp "
+" -background circle %i"%(self.stack['boxsize']/self.params['bin']*0.4)
+" -remove_black_dust"
+" -remove_white_dust"
+" -thr_black_dust -%.2f"%(self.params['xmipp-norm'])
+" -thr_white_dust %.2f"%(self.params['xmipp-norm'])
)
xmippcmd = xmippexe+" "+xmippopts
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
normtime = time.time() - normtime
apDisplay.printMsg("Xmipp normalization time: "+apDisplay.timeString(normtime))
### recombine particles to a single imagic stack
tmpstack = "tmp.xmippStack.hed"
apXmipp.gatherSingleFilesIntoStack(selfile,tmpstack)
apFile.moveStack(tmpstack,outStackPath)
### clean up directory
apFile.removeFile(selfile)
apFile.removeDir("partfiles")
def writeRawDataFile(self, eulertree):
#write to file
rawfile = "rawdata"+self.datastr+".dat"
apDisplay.printMsg("Writing raw data to file: "+rawfile)
r = open(rawfile, "w")
r.write("p1-id\tp1-e1\tp1-e2\tp1-e3\tmirror\treject\t"
+"p2-id\tp2-e1\tp2-e2\tp2-e3\tmirror\treject\t"
+"ang-dist\trot-dist\ttotal-dist\n")
for eulerpair in eulertree:
mystr = (
str(eulerpair['part1']['partid'])+"\t"+
str(round(eulerpair['part1']['euler1'],2))+"\t"+
str(round(eulerpair['part1']['euler2'],2))+"\t"+
str(round(eulerpair['part1']['euler3'],2))+"\t"+
str(eulerpair['part1']['mirror'])+"\t"+
str(eulerpair['part1']['reject'])+"\t"+
str(eulerpair['part2']['partid'])+"\t"+
str(round(eulerpair['part2']['euler1'],2))+"\t"+
str(round(eulerpair['part2']['euler2'],2))+"\t"+
str(round(eulerpair['part2']['euler3'],2))+"\t"+
str(eulerpair['part2']['mirror'])+"\t"+
str(eulerpair['part2']['reject'])+"\t"+
#str(round(eulerpair['angdist'],2))+"\t"+
str(round(eulerpair['rotdist'],2))+"\t"+
str(round(eulerpair['totdist'],2))+"\n"
)
r.write(mystr)
r.close()
return
def removeOverlappingPeaks(peaktree, cutoff, msg=True, doubles=False):
#distance in pixels for two peaks to be too close together
if msg is True:
apDisplay.printMsg("overlap distance cutoff: "+str(round(cutoff,1))+" pixels")
cutsq = cutoff**2 + 1
initpeaks = len(peaktree)
#orders peaks from smallest to biggest
peaktree.sort(_peakCompareSmallBig)
doublepeaktree = []
i=0
while i < len(peaktree):
j = i+1
while j < len(peaktree):
distsq = peakDistSq(peaktree[i], peaktree[j])
if(distsq < cutsq):
doublepeaktree.append(peaktree[j])
del peaktree[i]
i -= 1
j = len(peaktree)
j += 1
i += 1
if doubles is True:
peaktree = removeOverlappingPeaks(doublepeaktree, cutoff, False, False)
numpeaks = len(peaktree)
if msg is True:
apDisplay.printMsg("kept "+str(numpeaks)+" non-overlapping peaks of "
+str(initpeaks)+" total peaks")
return peaktree
def findPeaks(imgdict, maplist, params, maptype="ccmaxmap", pikfile=True):
peaktreelist = []
count = 0
imgname = imgdict['filename']
mapdir = os.path.join(params['rundir'], "maps")
thresh = float(params["thresh"])
bin = int(params["bin"])
diam = float(params["diam"])
apix = float(params["apix"])
olapmult = float(params["overlapmult"])
maxpeaks = int(params["maxpeaks"])
maxthresh = params["maxthresh"]
maxsizemult = float(params["maxsize"])
peaktype = params["peaktype"]
msg = not params['background']
pixdiam = diam/apix/float(bin)
pixrad = diam/apix/2.0/float(bin)
tmpldbid = None
mapdiam = None
for imgmap in maplist:
count += 1
if 'templateIds' in params:
#template correlator
tmpldbid = params['templateIds'][count-1]
elif 'diamarray' in params:
#dogpicker
mapdiam = params['diamarray'][count-1]
#find peaks
peaktree = findPeaksInMap(imgmap, thresh, pixdiam, count, olapmult,
maxpeaks, maxsizemult, msg, tmpldbid, mapdiam, bin=bin, peaktype=peaktype)
#remove border peaks
peaktree = removeBorderPeaks(peaktree, pixdiam, imgdict['image'].shape[1], imgdict['image'].shape[0])
#write map to jpeg with highlighted peaks
outfile = os.path.join(mapdir, imgname+"."+maptype+str(count)+".jpg")
createPeakMapImage(peaktree, imgmap, outfile, pixrad, bin, msg)
#write pikfile
if pikfile is True:
peakTreeToPikFile(peaktree, imgname, count, params['rundir'])
#append to complete list of peaks
peaktreelist.append(peaktree)
peaktree = mergePeakTrees(imgdict, peaktreelist, params, msg, pikfile=pikfile)
#max threshold
if maxthresh is not None:
precount = len(peaktree)
peaktree = maxThreshPeaks(peaktree, maxthresh)
postcount = len(peaktree)
#if precount != postcount:
apDisplay.printMsg("Filtered %d particles above threshold %.2f"%(precount-postcount,maxthresh))
return peaktree
def getNumAlignedParticles(self):
t0 = time.time()
self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
oldalignedstack = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['imagicfile'])
numpart = apFile.numImagesInStack(oldalignedstack)
apDisplay.printMsg("numpart="+str(numpart)+" in "+apDisplay.timeString(time.time()-t0))
return numpart
def fillSimilarityMatrix(self, alignedstack):
### Get initial correlation values
### this is really, really slow
similarfile = "similarities.dat"
simstack = similarityStack()
simstack.similarfile = similarfile
simstack.start(alignedstack)
if not os.path.isfile(similarfile):
apDisplay.printError("Failed to create similarity file")
simf = open(similarfile, 'r')
simlist = []
count = 0
for line in simf:
count += 1
sline = line.strip()
slist = sline.split()
ccval = float(slist[2])
simlist.append(ccval)
simf.close()
apDisplay.printMsg("There are %d lines in the sim file: %s"%(count, similarfile))
numpart = apFile.numImagesInStack(alignedstack)
if count != numpart*(numpart-1):
### we have a valid file already
apDisplay.printError("There are only %d lines need to have %d"%(count, numpart*(numpart-1)))
return similarfile, simlist
def calcResolution(self, partlist, stackfile, apix):
### group particles by refnum
reflistsdict = {}
for partdict in partlist:
refnum = partdict['template']
partnum = partdict['num']
if not refnum in reflistsdict:
reflistsdict[refnum] = []
reflistsdict[refnum].append(partnum)
### get resolution
self.resdict = {}
boxsizetuple = apFile.getBoxSize(stackfile)
boxsize = boxsizetuple[0]
for refnum in reflistsdict.keys():
partlist = reflistsdict[refnum]
esttime = 3e-6 * len(partlist) * boxsize**2
apDisplay.printMsg("Ref num %d; %d parts; est time %s"
%(refnum, len(partlist), apDisplay.timeString(esttime)))
frcdata = apFourier.spectralSNRStack(stackfile, apix, partlist, msg=False)
frcfile = "frcplot-%03d.dat"%(refnum)
apFourier.writeFrcPlot(frcfile, frcdata, apix, boxsize)
res = apFourier.getResolution(frcdata, apix, boxsize)
self.resdict[refnum] = res
return
def parseParticleDataIterationFile(self,paramfile,test=False):
'''
parse data.star file from Relion 1.2:
'''
if not os.path.isfile(paramfile):
apDisplay.printError("Relion data.star file does not exist: %s" % (paramfile))
apDisplay.printMsg("Parsing parameter file: %s" % (paramfile))
# Use the star file class to read the parameter file
f = starFile.StarFile( paramfile )
f.read()
dataBlock = f.getDataBlock("data_images")
loopDict = dataBlock.getLoopDict() # there is only one loop in the data_images block
partnum = 1 # partnum starts with 1, not 0
partdict = {}
for valueSet in loopDict:
paramdict = {
'partnum' : partnum,
'angleRot' : float(valueSet["_rlnAngleRot"]),
'angleTilt' : float(valueSet["_rlnAngleTilt"]),
'anglePsi' : float(valueSet["_rlnAnglePsi"]),
'originX' : float(valueSet["_rlnOriginX"]),
'originY' : float(valueSet["_rlnOriginY"]),
}
partdict[paramdict['partnum']] = paramdict
partnum = partnum + 1
if len(partdict) < 2:
apDisplay.printError("No particles found in particle data file %s" % (paramfile))
apDisplay.printMsg("Processed %d particles" % (len(partdict)))
return partdict
def getGoodParticles(self, classpartdatas, norefclassnum):
includeParticle = []
tiltParticlesData = []
nopairParticle = 0
excludeParticle = 0
apDisplay.printMsg("sorting particles")
for classpart in classpartdatas:
#write to text file
classnum = classpart['classNumber']-1
if classnum == norefclassnum:
notstackpartnum = classpart['noref_particle']['particle']['particleNumber']
tiltstackpartdata = apTiltPair.getStackParticleTiltPair(self.params['notstackid'],
notstackpartnum, self.params['tiltstackid'])
if tiltstackpartdata is None:
nopairParticle += 1
else:
emantiltstackpartnum = tiltstackpartdata['particleNumber']-1
includeParticle.append(emantiltstackpartnum)
tiltParticlesData.append(tiltstackpartdata)
else:
excludeParticle += 1
includeParticle.sort()
apDisplay.printMsg("Keeping "+str(len(includeParticle))+" and excluding \n\t"
+str(excludeParticle)+" particles with "+str(nopairParticle)+" unpaired particles")
if len(includeParticle) < 1:
apDisplay.printError("No particles were kept")
return includeParticle, tiltParticlesData
def insertAnalysis(self, imagicstack, runtime, insert=False):
### create MSAParam object
msaq = appiondata.ApImagicAlignAnalysisData()
msaq['runname'] = self.params['runname']
msaq['run_seconds'] = runtime
msaq['bin'] = self.params['bin']
msaq['highpass'] = self.params['hpfilt']
msaq['lowpass'] = self.params['lpfilt']
msaq['mask_radius'] = self.params['mask_radius']
msaq['mask_dropoff'] = self.params['mask_dropoff']
msaq['numiters'] = self.params['numiters']
msaq['overcorrection'] = self.params['overcorrection']
msaq['MSAdistance'] = self.params['MSAdistance']
msaq['eigenimages'] = "eigenimages"
### finish analysis run
analysisrunq = appiondata.ApAlignAnalysisRunData()
analysisrunq['runname'] = self.params['runname']
analysisrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
analysisrunq['imagicMSArun'] = msaq
analysisrunq['alignstack'] = self.alignstackdata
analysisrunq['hidden'] = False
analysisrunq['description'] = self.params['description']
apDisplay.printMsg("inserting Align Analysis Run parameters into database")
if insert is True:
analysisrunq.insert()
return
def readFile(self, oldmrcfile):
apDisplay.printMsg('Reading %s into memory' % oldmrcfile)
imagearray = mrc.read(oldmrcfile)
# invert image density
if self.params['invert'] is True:
imagearray *= -1.0
return imagearray
def getTransformImageIds(transformdata):
t0 = time.time()
img1 = transformdata.special_getitem('image1', dereference=False).dbid
img2 = transformdata.special_getitem('image2', dereference=False).dbid
if time.time()-t0 > 0.3:
apDisplay.printMsg("long image query "+apDisplay.timeString(time.time()-t0))
return img1, img2
def insertFSCData(self, iternum, refineIterData):
fscfile = 'fsc.eotest.%d'%(iternum)
fscpath = os.path.join(self.params['rundir'], "iter%03d"%(iternum), fscfile)
if not os.path.isfile(fscpath):
apDisplay.printWarning("Could not find FSC file: "+fscpath)
return None
f = open(fscpath, 'r')
apDisplay.printMsg("inserting FSC Data into database")
numinserts = 0
for line in f:
fscq = appiondata.ApFSCData()
fscq['refineIter'] = refineIterData
sline = line.strip()
bits = sline.split('\t')
fscq['pix'] = int(bits[0])
fscq['value'] = float(bits[1])
numinserts+=1
if self.params['commit'] is True:
fscq.insert()
apDisplay.printMsg("inserted "+str(numinserts)+" rows of FSC data into database")
f.close()
def getParticleTiltRotationAnglesOTR(stackpartdata):
partdata = stackpartdata['particle']
imgnum, transformdata, otherpartdata = getTiltTransformFromParticle(partdata)
t0 = time.time()
tiltangle1, tiltangle2 = apDatabase.getTiltAnglesDegFromTransform(transformdata)
if time.time()-t0 > 1.0:
apDisplay.printMsg("long angle query "+apDisplay.timeString(time.time()-t0))
if imgnum == 1:
### negative case, tilt picker theta < 0
tiltrot = transformdata['image1_rotation']
theta = transformdata['tilt_angle']
notrot = transformdata['image2_rotation']
tiltangle = tiltangle1 - tiltangle2
elif imgnum == 2:
### positive case, tilt picker theta > 0
tiltrot = transformdata['image2_rotation']
theta = transformdata['tilt_angle']
notrot = transformdata['image1_rotation']
tiltangle = tiltangle2 - tiltangle1
else:
#no particle pair info was found or some other problem
print partdata
apDisplay.printError("failed to get tilt pair data or some other problem")
if transformdata.timestamp < datetime.datetime(2009, 2, 19, 0, 0, 0):
### bugfix for switched tilt axis angles, before Feb 19, 2009
#apDisplay.printWarning("Switching angles")
temprot = notrot
notrot = tiltrot
tiltrot = temprot
#print "tr=%.2f, th=%.2f, nr=%.2f, tilt=%.2f"%(tiltrot, theta, notrot, tiltangle)
return tiltrot, theta, notrot, tiltangle
def onAdd(self, evt):
vertices = []
vertices = self.panel.getTargetPositions('Region to Remove')
apDisplay.printMsg("Removing region contained in %d polygon vertices"%(len(vertices)))
def reversexy(coord):
clist=list(coord)
clist.reverse()
return tuple(clist)
vertices = map(reversexy,vertices)
maskimg = leginon.polygon.filledPolygon(self.panel.imagedata.shape,vertices)
type(maskimg)
for label in self.labels:
targets = self.panel.getTargets(label)
eliminated = 0
newparticles = []
for target in targets:
coord = (target.y,target.x)
if maskimg[coord] != 0:
eliminated += 1
else:
newparticles.append(target)
apDisplay.printMsg("%d particle(s) eliminated due to masking"%(eliminated))
self.panel.setTargets(label,newparticles)
self.panel.setTargets('Region to Remove', [])
def runKerdenSOM(self, indata):
"""
From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/KerDenSOM
KerDenSOM stands for "Kernel Probability Density Estimator Self-Organizing Map".
It maps a set of high dimensional input vectors into a two-dimensional grid.
"""
apDisplay.printMsg("Running KerDen SOM")
outstamp = os.path.join(self.params['rundir'], self.timestamp)
kerdencmd = ( "xmipp_classify_kerdensom -verb 1 -i %s -o %s -xdim %d -ydim %d -saveclusters "%
(indata, outstamp, self.params['xdim'], self.params['ydim'])
)
### convergence criteria
if self.params['converge'] == "fast":
kerdencmd += " -eps 1e-5 "
elif self.params['converge'] == "slow":
kerdencmd += " -eps 1e-9 "
else:
kerdencmd += " -eps 1e-7 "
apDisplay.printColor(kerdencmd, "cyan")
proc = subprocess.Popen(kerdencmd, shell=True)
proc.wait()
time.sleep(1)
return
def start(self):
aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
boxsize = aligndata['boxsize']
apix = aligndata['pixelsize']
maskpixrad = self.params['maskrad']/apix
if maskpixrad*2 > boxsize-2:
apDisplay.printError("Mask radius is too big for boxsize: %d > %d"%(maskpixrad*2,boxsize-2))
apDisplay.printMsg("Mask radius and boxsize: %.1f < %d"%(maskpixrad*2,boxsize-2))
self.instack = os.path.join(aligndata['path']['path'], aligndata['imagicfile'])
outdata = "stack.data"
apXmipp.convertStackToXmippData(self.instack, outdata, maskpixrad,
boxsize, numpart=self.params['numpart']-1)
self.runKerdenSOM(outdata)
if apFile.stackSize(self.instack) > 3.0*(1024**3):
# Big stacks use eman
self.createMontageByEMAN()
binned = None
else:
binned = self.createMontageInMemory(apix)
self.insertKerDenSOM(binned=binned)
apFile.removeFile(outdata)
apFile.removeFilePattern("*.cod")
def loopProcessImage(self, imgdata):
#creates self.peaktree and filterLoop sets self.filtarray
self.peaktree = filterLoop.FilterLoop.loopProcessImage(self, imgdata)
if self.params['background'] is False:
apDisplay.printMsg("Found "+str(len(self.peaktree))+" particles for "
+apDisplay.shortenImageName(imgdata['filename']))
self.stats['lastpeaks'] = len(self.peaktree)
if self.params['nojpegs'] is False:
if self.threadJpeg is True:
threading.Thread(target=apPeaks.createPeakJpeg, args=(imgdata, self.peaktree, self.params, self.filtarray)).start()
else:
apPeaks.createPeakJpeg(imgdata, self.peaktree, self.params, self.filtarray)
elif self.params['background'] is False:
apDisplay.printWarning("Skipping JPEG creation")
if self.params['defocpair'] is True:
self.sibling, self.shiftpeak = apDefocalPairs.getShiftFromImage(imgdata, self.params['sessionname'])
return
def createMontageInMemory(self):
apDisplay.printMsg("Converting files")
#logging.debug('Inside createMontageInMemory')
montagepngs = []
files = glob.glob('spectra_??_??.png')
for pngfile in files:
montagepngs.append(pngfile)
### create montage
apFile.removeFile("montage.png")
montagecmd = "montage -geometry +4+4 -tile %dx%d " % (
self.params['xdim'], self.params['ydim'])
for monpng in montagepngs:
montagecmd += monpng + " "
montagecmd += "montage.png"
#logging.debug('montagecmd ' + montagecmd)
proc = subprocess.Popen(montagecmd, shell=True)
proc.wait()
time.sleep(1)
def readRefDocFile(self):
reflist = []
docfile = "ref" + self.params['timestamp'] + ".doc"
if not os.path.isfile(docfile):
apDisplay.printError("could not find doc file " + docfile +
" to read reference angles")
f = open(docfile, "r")
mininplane = 360.0
for line in f:
if line[:2] == ' ;':
continue
spidict = operations.spiderInLine(line)
refdict = self.spidict2partdict(spidict)
if refdict['inplane'] < mininplane:
mininplane = refdict['inplane']
reflist.append(refdict)
for refdict in reflist:
refdict['inplane'] = refdict['inplane'] - mininplane
apDisplay.printMsg("read rotation and shift parameters for " +
str(len(reflist)) + " references")
return reflist
def insertFSC(self, fscfile, refineIterData, commit=True):
if not os.path.isfile(fscfile):
apDisplay.printWarning("Could not open FSC file: " + fscfile)
f = open(fscfile, 'r')
apDisplay.printMsg("inserting FSC Data into database")
numinserts = 0
for line in f:
fscq = appiondata.ApFSCData()
fscq['refineIter'] = refineIterData
line = line.rstrip()
bits = line.split('\t')
fscq['pix'] = int(bits[0])
fscq['value'] = float(bits[1])
numinserts += 1
if commit is True:
fscq.insert()
apDisplay.printMsg("inserted " + str(numinserts) +
" rows of FSC data into database")
f.close()
def getTimestamp(self):
timestamp = None
if self.params['timestamp'] is not None:
return self.params['timestamp']
if self.params["jobid"] is not None:
jobdata = appiondata.ApSparxISACJobData.direct_query(
self.params["jobid"])
timestamp = jobdata['timestamp']
elif timestamp is None:
wildcard = "isac-*-params.pickle"
files = glob.glob(wildcard)
if len(files) == 0:
apDisplay.printError("Could not determine timestamp\n" +
"please provide it, e.g. -t 08nov27e54")
reg = re.match("isac-([0-9a-z]*)-", files[0])
if len(reg.groups()) == 0:
apDisplay.printError("Could not determine timestamp\n" +
"please provide it, e.g. -t 08nov27e54")
timestamp = reg.groups()[0]
apDisplay.printMsg("Found timestamp = '" + timestamp + "'")
return timestamp
def normalLeastSquares(X, Y):
"""
solve using the normal equations with no manipulation
"""
t0 = time.time()
### check the input
if checkMatrixSizes(X, Y) is False:
return None
### solve it
XT = numpy.transpose(X)
# create a square matrix
XTX = numpy.dot(XT, X)
if numpy.linalg.det(XTX) == 0:
apDisplay.printWarning("Singular matrix in calculation")
return None
XTXinv = numpy.linalg.inv(XTX)
beta = numpy.dot(numpy.dot(XTXinv, XT), Y)
apDisplay.printMsg("normalLeastSquares completed in %s" %
(apDisplay.timeString(time.time() - t0)))
return beta
def uploadZProjection(runname, initialimagedata, uploadfile):
presetdata = leginon.leginondata.PresetData(
initializer=initialimagedata['preset'])
presetdata['name'] = 'Zproj'
imagedata = leginon.leginondata.AcquisitionImageData(
initializer=initialimagedata)
basename = os.path.basename(uploadfile)
splitname = os.path.splitext(basename)
names = splitname[0].split('_zproject')
projectionname = names[0] + '_' + runname + '_zproject'
imagedata['filename'] = projectionname
imagedata['label'] = 'projection'
imagedata['preset'] = presetdata
newimgfilepath = os.path.join(imagedata['session']['image path'],
projectionname)
apDisplay.printMsg("Copying original image to a new location: " +
newimgfilepath)
shutil.copyfile(uploadfile, newimgfilepath)
image = mrc.read(newimgfilepath)
imagedata['image'] = image
return publish(imagedata)
def start(self):
self.runtime = 0
self.checkAffPropRun()
self.numpart = self.getNumAlignedParticles()
alignedstack = self.prepareStack()
### run Affinity Propagation
aptime = time.time()
classes = self.runAffinityPropagation(alignedstack)
aptime = time.time() - aptime
### insert into database
inserttime = time.time()
self.runtime = aptime
self.insertAffinityPropagationRun(classes)
inserttime = time.time() - inserttime
apDisplay.printMsg("Affinity propagation time: " +
apDisplay.timeString(aptime))
apDisplay.printMsg("Database Insertion time: " +
apDisplay.timeString(inserttime))
def convertSQLtoTree(results):
t0 = time.time()
parttree = []
for row in results:
if len(row) < 3:
apDisplay.printError("delete MySQL cache file and run again")
try:
if row[2] < 1.0:
partpair1 = { 'part1': int(row[0]), 'part2': int(row[1]), 'tilt': False }
partpair2 = { 'part1': int(row[1]), 'part2': int(row[0]), 'tilt': True }
else:
partpair1 = { 'part1': int(row[0]), 'part2': int(row[1]), 'tilt': True }
partpair2 = { 'part1': int(row[1]), 'part2': int(row[0]), 'tilt': False }
parttree.append(partpair1)
parttree.append(partpair2)
except:
print row
apDisplay.printError("bad row entry")
apDisplay.printMsg("Converted "+str(len(parttree))+" particles in "+apDisplay.timeString(time.time()-t0))
return parttree
def getSymmetryDataFromName(symtext='c1', msg=True):
# find the symmetry entry (full symmetry name) in the database
if len(symtext.split()) > 1:
symdataq = appiondata.ApSymmetryData(symmetry=symtext.strip())
else:
# based on the text version from EMAN
# first convert to lower case
symtext = symtext.lower().strip()
if symtext == "i":
symtext = "icos"
symdataq = appiondata.ApSymmetryData(eman_name=symtext)
symdatas = symdataq.query()
if not symdatas:
apDisplay.printError("No symmetry named %s was found" % (symtext))
# select oldest match
symdata = symdatas[len(symdatas) - 1]
if msg is True:
apDisplay.printMsg("Selected symmetry group: " + apDisplay.colorString(
"%s -- %s" %
(symdata['eman_name'].upper(), symdata['symmetry']), "cyan"))
return symdata
def stackHDFToIMAGIC(hdfFile, imagicFile=None):
"""
convert HDF stack back into an IMAGIC stack
"""
numPart = EMAN2.EMUtil.get_image_count(hdfFile)
# output must end with hdf
root, ext = os.path.splitext(hdfFile)
if imagicFile is None or ext != ".hed":
imagicFile = root + ".hed"
apFile.removeFile(imagicFile)
apFile.removeFile(root + ".img")
apDisplay.printMsg("Creating IMAGIC file '%s' with %d particles" %
(imagicFile, numPart))
headerOnly = False
for i in range(numPart):
imgData = EMAN2.EMData.read_images(hdfFile, [i], headerOnly)[0]
imgData.write_image(imagicFile, i)
return imagicFile
def runSatAverage(self):
keepfile = os.path.join(self.params['rundir'], "keeplist-tot"+self.datastr+".lst")
newname = "recon%d_cut%d_iter%d.hed" % (self.params['reconid'], self.params['cutrange']*10, self.iternum)
volname = "recon%d_cut%d_iter%d.%da.mrc"% (self.params['reconid'], self.params['cutrange']*10, self.iternum, self.iternum)
volfile = os.path.join(self.params['rundir'], "volumes", volname)
if os.path.isfile(volfile):
apDisplay.printMsg("Skipping sat average, volume exists")
return
cmd = ( "satAverage.py "
+" --projectid="+str(self.params['projectid'])
+" --reconid="+str(self.params['reconid'])
+" \\\n --mask=40 --iter="+str(self.iternum)
+" \\\n --stackname="+newname
+" \\\n --keep-list="+keepfile
+" \n" )
print "New satAverage.py Command:"
apDisplay.printColor(cmd, "purple")
if self.params['sataverage'] is True:
proc = subprocess.Popen(cmd, shell=True)
proc.communicate()
return
def getImageIdsFromStack(stackid, msg=True):
if stackid < 1:
return []
t0 = time.time()
stackdata = appiondata.ApStackData.direct_query(stackid)
stackq = appiondata.ApStackParticleData()
stackq['stack'] = stackdata
stackparticledata = stackq.query()
stackimages = []
if msg is True:
apDisplay.printMsg("querying particle images from stackid=" +
str(stackid) + " on " + time.asctime())
for sp in stackparticledata:
spimagedata = sp['particle']['image']
spimageid = spimagedata.dbid
if spimageid not in stackimages:
stackimages.append(spimageid)
if msg is True:
apDisplay.printMsg("Found %d images from stackid=%d" %
(len(stackimages), stackid))
return stackimages
def checkStackNumbering(stackname):
import EMAN
# check that the numbering is stored in the NImg parameter
apDisplay.printMsg("checking that original stack is numbered")
n = EMAN.fileCount(stackname)[0]
im = EMAN.EMData()
im.readImage(stackname, n - 1)
# if last particle is not numbered with same value as # of particles,
# renumber the entire stack
if n - 1 != im.NImg():
apDisplay.printWarning(
"Original stack is not numbered! numbering now...")
# check that the stack exists in a writable directory and is not read-only
if os.access(stackname, os.W_OK) is True:
numberParticlesInStack(stackname, startnum=0, verbose=True)
else:
apDisplay.printWarning(
"old stack header exists in a READ-only directory and cannot be numbered according to EMAN"
)
return
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['description'] is None:
apDisplay.printError("run description was not defined")
if self.params['templatelist'] is None:
apDisplay.printError("template list was not provided")
if self.params['lastring'] is None:
apDisplay.printError("a last ring radius was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles " +
str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(
self.params['stackid']) / self.params['bin']
if self.params['lastring'] > boxsize / 2 - 2:
apDisplay.printError("last ring radius is too big for boxsize " +
str(self.params['lastring']) + " > " +
str(boxsize / 2 - 2))
if self.params['lastring'] + self.params['xysearch'] > boxsize / 2 - 2:
apDisplay.printError(
"last ring plus xysearch radius is too big for boxsize " +
str(self.params['lastring'] + self.params['xysearch']) +
" > " + str(boxsize / 2 - 2))
### convert / check template data
self.templatelist = self.params['templatelist'].strip().split(",")
if not self.templatelist or type(self.templatelist) != type([]):
apDisplay.printError("could not parse template list=" +
self.params['templatelist'])
self.params['numtemplate'] = len(self.templatelist)
apDisplay.printMsg("Found " + str(self.params['numtemplate']) +
" templates")
def alignPicks2(picks1, picks2, data, limit=20.0, msg=True):
### create distance dictionary
alignpicks2 = a2Toa1Data(picks2, data)
sortedDict2 = {}
index = 0
while index < len(alignpicks2):
p2 = alignpicks2[index]
key = "%d,%d" % (
p2[0] / limit,
p2[1] / limit,
)
if not key in sortedDict2:
sortedDict2[key] = [
index,
]
else:
sortedDict2[key].append(index)
index += 1
### find matching picks
filled = {}
list1 = []
alignlist2 = []
for p1 in picks1:
closepick, dist = findClosestPick2(p1, alignpicks2, sortedDict2, limit)
if dist < limit:
key = str(closepick)
if not key in filled:
list1.append(p1)
alignlist2.append(closepick)
filled[key] = True
#convert lists
nlist1 = numpy.array(list1, dtype=numpy.int32)
nalignlist2 = numpy.array(alignlist2, dtype=numpy.int32)
#transform back
nlist2 = a1Toa2Data(nalignlist2, data)
if msg is True:
apDisplay.printMsg("Aligned "+str(len(nlist1))+" of "+str(len(picks1))+\
" particles to "+str(len(nlist2))+" of "+str(len(picks2)))
return nlist1, nlist2
def sortFolder(self):
numsort = 0
apDisplay.printMsg("Sorting files into clean folders")
### move files for all particle iterations
files = []
for i in range(self.lastiter+1):
iterdir = "iter%03d"%(i)
apParam.createDirectory(iterdir, warning=False)
wildcard = "part*_it*%03d_*.*"%(i)
files.extend(glob.glob(wildcard))
wildcard = "part*_it*%03d.*"%(i)
files.extend(glob.glob(wildcard))
for filename in files:
if os.path.isfile(filename):
numsort += 1
shutil.move(filename,iterdir)
if numsort < 3:
apDisplay.printWarning("Problem in iteration file sorting, are they already sorted?")
apDisplay.printMsg("Sorted "+str(numsort)+" iteration files")
### move files for all reference iterations
refsort = 0
refdir = "refalign"
apParam.createDirectory(refdir, warning=False)
wildcard = "ref*_it*.*"
files = glob.glob(wildcard)
for filename in files:
refsort += 1
shutil.move(filename, refdir)
#if refsort < 5:
# apDisplay.printError("Problem in reference file sorting")
apDisplay.printMsg("Sorted "+str(refsort)+" reference files")
return
def _removeProcessedImages(self):
startlen = len(self.imgtree)
donecount = 0
reproccount = 0
rejectcount = 0
self.stats['skipcount'] = 0
newimgtree = []
count = 0
t0 = time.time()
for imgdata in self.imgtree:
count += 1
if count % 10 == 0:
sys.stderr.write(".")
skip, reason = self.skipTestOnImage(imgdata)
imgname = imgdata['filename']
if skip is True:
if reason == 'reproc':
reproccount += 1
if skip is True:
if self.stats['skipcount'] == 0:
sys.stderr.write("skipping processed images\n")
elif self.stats['skipcount'] % 80 == 0:
sys.stderr.write(".\n")
else:
sys.stderr.write(".")
self.stats['skipcount'] += 1
else:
newimgtree.append(imgdata)
sys.stderr.write("\n")
apDisplay.printMsg("finished skipping in %s" %
(apDisplay.timeString(time.time() - t0)))
if self.stats['skipcount'] > 0:
self.imgtree = newimgtree
sys.stderr.write("\n")
apDisplay.printWarning("skipped " + str(self.stats['skipcount']) +
" of " + str(startlen) + " images")
apDisplay.printMsg("[[ " + str(reproccount) + " no reprocess " +
" | " + str(rejectcount) + " rejected " +
" | " + str(donecount) + " in donedict ]]")
def runMakeSpectra(self, indata):
"""
From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Makespectra
This program generates a Fourier-Bessel decomposition of each image listed in
a *.sel. Then creates a report ASCII file for each image, with the harmonic
energy percentage as a function of the radius, with extension .SPT.
"""
spectratime = time.time()
apDisplay.printMsg("Running make_spectra")
#logging.debug('Inside make_spectra')
#ROB
tempFileNameforSpectra = "tempFileNameforSpectra.txt"
tmpSelFile = apXmipp.breakupStackIntoSingleFiles(
indata, numpart=self.params['numpart'])
makespectracmd = ("xmipp_make_spectra -i %s -o %s \
-x0 %f -y0 %f -r1 %d -r2 %d -low %d -high %d " % (
tmpSelFile,
tempFileNameforSpectra,
self.xOffset,
self.yOffset,
self.params['spectrainnerradius'],
self.params['spectraouterradius'],
self.params['spectralowharmonic'],
self.params['spectrahighharmonic'],
))
#logging.debug(makespectracmd)
apDisplay.printColor(makespectracmd, "cyan")
proc = subprocess.Popen(makespectracmd,
shell=True,
stdout=subprocess.PIPE)
proc.wait()
apDisplay.printMsg("Spectra calculation complete in %s" %
(apDisplay.timeString(time.time() - spectratime)))
time.sleep(1)
return tempFileNameforSpectra
def start(self):
"""
this is the main component of the script
where all the processing is done
"""
### initialize some variables
self.runq = None
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
apDisplay.printMsg("Pixel size: %.5f"%(self.apix))
self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
apDisplay.printMsg("Box size: %d"%(self.boxsize))
self.checkResults()
self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
self.numiter = self.getNumberOfIterations()
for i in range(self.numiter):
iternum = i+1
apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green")
self.uploadIteration(iternum)
reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir'])
if reconrunid:
apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid'])
apRecon.setGoodBadParticlesFromReconId(reconrunid)
else:
apDisplay.printWarning("Could not find recon run id")
def imageToArray(im, convertType='uint8', dtype=None, msg=True):
"""
Convert PIL image to numpy array
copied and modified from http://mail.python.org/pipermail/image-sig/2005-September/003554.html
"""
if im.mode == "L":
a = numpy.fromstring(im.tostring(), numpy.uint8)
a = numpy.reshape(a, (im.size[1], im.size[0]))
#a.shape = (im.size[1], im.size[0], 1) # alternate way
elif (im.mode == 'RGB'):
apDisplay.printMsg("reading RGB and converting to L")
grey = im.convert('L')
a = numpy.fromstring(grey.tostring(), numpy.uint8)
a = numpy.reshape(a, (grey.size[1], grey.size[0]))
elif (im.mode == 'RGBA'):
apDisplay.printMsg("reading RGBA and converting to L")
grey = im.convert('L')
a = numpy.fromstring(grey.tostring(), numpy.uint8)
a = numpy.reshape(a, (grey.size[1], grey.size[0]))
elif (im.mode == 'LA'):
apDisplay.printMsg("reading LA and converting to L")
grey = im.convert('L')
a = numpy.fromstring(grey.tostring(), numpy.uint8)
a = numpy.reshape(a, (grey.size[1], grey.size[0]))
else:
raise ValueError, im.mode + " mode not considered"
if convertType == 'float32':
a = a.astype(numpy.float32)
if dtype is not None:
a = a.astype(dtype)
return a
def importPicks(self, picks1, picks2, tight=False, msg=True):
t0 = time.time()
#print picks1
#print self.currentpicks1
curpicks1 = numpy.asarray(self.currentpicks1)
curpicks2 = numpy.asarray(self.currentpicks2)
#print curpicks1
# get picks
apTiltTransform.setPointsFromArrays(curpicks1, curpicks2, self.data)
pixdiam = self.data['pixdiam']
if tight is True:
pixdiam /= 4.0
#print self.data, pixdiam
list1, list2 = apTiltTransform.alignPicks2(picks1,
picks2,
self.data,
limit=pixdiam,
msg=msg)
if list1.shape[0] == 0 or list2.shape[0] == 0:
apDisplay.printWarning("No new picks were found")
# merge picks
newpicks1, newpicks2 = apTiltTransform.betterMergePicks(curpicks1,
list1,
curpicks2,
list2,
msg=msg)
newparts = newpicks1.shape[0] - curpicks1.shape[0]
# copy over picks
self.currentpicks1 = newpicks1
self.currentpicks2 = newpicks2
if msg is True:
apDisplay.printMsg("Inserted " + str(newparts) +
" new particles in " +
apDisplay.timeString(time.time() - t0))
return True
def getPartcileLists(self):
#first query
query1 = self.queryParticles(swap=False)
self.cursor.execute(query1)
results1 = self.cursor.fetchall()
apDisplay.printMsg("Found "+str(len(results1))+" particle pairs in forward order")
#if len(results1) < 2:
# apDisplay.printError("Failed to find any particles")
#swap particle1 and particle2 in ApTiltParticlePairData
query2 = self.queryParticles(swap=True)
self.cursor.execute(query2)
results2 = self.cursor.fetchall()
apDisplay.printMsg("Found "+str(len(results2))+" particle pairs in reverse order")
#if len(results2) < 2:
# apDisplay.printError("Failed to find any particles")
parttree = self.parseResults(results1, results2)
f = open("tiltsync-"+self.timestamp+".dat", "w")
count = 0
for partdict in parttree:
count += 1
emannotnum = partdict['not']-1
emantiltnum = partdict['tilt']-1
line = operations.spiderOutLine(count, (emannotnum,emantiltnum))
f.write(line)
f.close()
apDisplay.printMsg("Writing "+str(len(parttree))+" particle pairs")
if len(parttree) < 2:
apDisplay.printError("Failed to find any particle pairs")
return parttree
def getResolutionFromFSCFile(fscfile, boxsize, apix, msg=False):
"""
should use more general apFourier.getResolution()
"""
if not os.path.isfile(fscfile):
apDisplay.printError("fsc file does not exist")
if msg is True:
apDisplay.printMsg("box: %d, apix: %.3f, file: %s" %
(boxsize, apix, fscfile))
f = open(fscfile, 'r')
lastx = 0
lasty = 0
for line in f:
xy = line.strip().split()
x = float(xy[0])
y = float(xy[1])
if x != 0.0 and x < 0.9:
apDisplay.printWarning("FSC is wrong data format")
if y > 0.5:
#store values for later
lastx = x
lasty = y
else:
# get difference of fsc
diffy = lasty - y
# get distance from 0.5
distfsc = (0.5 - y) / diffy
# get interpolated spatial freq
intfsc = x - distfsc * (x - lastx)
# convert to Angstroms
if intfsc > 0.0:
res = boxsize * apix / intfsc
else:
res = boxsize * apix
f.close()
return res
# fsc did not fall below 0.5
apDisplay.printWarning("Failed to determine resolution")
res = boxsize * apix / (lastx + 1)
return res
def optimizeAngles(self, msg=True):
t0 = time.time()
### run find theta
na1 = numpy.array(self.currentpicks1, dtype=numpy.int32)
na2 = numpy.array(self.currentpicks2, dtype=numpy.int32)
# minimum area for a triangle to be valid
arealim = 100.0
fittheta = radermacher.tiltang(na1, na2, arealim)
if not fittheta or not 'wtheta' in fittheta:
return
theta = fittheta['wtheta']
thetadev = fittheta['wthetadev']
if msg is True:
thetastr = ("%3.3f +/- %2.2f" % (theta, thetadev))
tristr = apDisplay.orderOfMag(
fittheta['numtri']) + " of " + apDisplay.orderOfMag(
fittheta['tottri'])
tristr = (" (%3.1f " % (100.0 * fittheta['numtri'] /
float(fittheta['tottri']))) + "%) "
apDisplay.printMsg("Tilt angle " + thetastr + tristr)
self.data['theta'] = fittheta['wtheta']
### run optimize angles
lastiter = [80, 80, 80]
count = 0
totaliter = 0
while max(lastiter) > 75 and count < 30:
count += 1
lsfit = self.runLeastSquares()
lastiter[2] = lastiter[1]
lastiter[1] = lastiter[0]
lastiter[0] = lsfit['iter']
totaliter += lsfit['iter']
if msg is True:
apDisplay.printMsg("Least Squares: " + str(count) +
" rounds, " + str(totaliter) +
" iters, rmsd of " +
str(round(lsfit['rmsd'], 4)) +
" pixels in " +
apDisplay.timeString(time.time() - t0))
return
def start(self):
parttree = apFrealign.parseFrealignParamFile(self.params['paramfile'])
phases = []
for partdict in parttree:
phases.append(partdict['phase_residual'])
del parttree
phases.sort()
phases = numpy.array(phases, dtype=numpy.float32)
if self.params['gracex'] is None:
apDisplay.printMsg("xFile: %s" % (self.params['paramfile']))
apDisplay.printMsg("xMean: %.3f +/- %.3f (%d total)" %
(phases.mean(), phases.std(), phases.shape[0]))
apDisplay.printMsg(
"xRange: %.2f (min) <> %.2f (0.1) <> %.2f (0.25) <> %.2f (med) <> %.2f (0.75) <> %.2f (0.9) <> %.2f (max)"
% (phases.min(), phases[int(phases.shape[0] * 0.1)],
phases[int(phases.shape[0] * 0.25)], numpy.median(phases),
phases[int(phases.shape[0] * 0.75)], phases[int(
phases.shape[0] * 0.9)], phases.max()))
else:
f = open("boxplot.dat", "a")
#boxplot line: (X, median, upper/lower limit, upper/lower whisker)
f.write("%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n" % (
self.params['gracex'],
numpy.median(phases),
phases[int(phases.shape[0] * 0.75)],
phases[int(phases.shape[0] * 0.25)],
phases[int(phases.shape[0] * 0.9)],
phases[int(phases.shape[0] * 0.1)],
))
f.close()
f = open("means.dat", "a")
f.write("%d\t%.8f\n" % (self.params['gracex'], phases.mean()))
f.close()
def loadOneRawFrame(self, rawframe_path, frame_number):
'''
Load one raw frame depending on the rawframetype
'''
if self.getRawFrameType() == 'singles':
return super(GatanK2Processing,
self).loadOneRawFrame(rawframe_path, frame_number)
'''
Load from rawframe_path (a stack file) the chosen frame of the current image.
'''
try:
bin = self.camerainfo['binning']
offset = self.camerainfo['offset']
dimension = self.camerainfo['dimension']
except:
# default
bin = {'x': 1, 'y': 1}
offset = {'x': 0, 'y': 0}
dimension = self.getDefaultDimension()
crop_end = {
'x': offset['x'] + dimension['x'] * bin['x'],
'y': offset['y'] + dimension['y'] * bin['y']
}
apDisplay.printMsg('Frame path: %s' % rawframe_path)
waitmin = 0
while not os.path.exists(rawframe_path):
if self.waittime < 0.1:
apDisplay.printWarning('Frame File %s does not exist.' %
rawframe_path)
return False
apDisplay.printWarning(
'Frame File %s does not exist. Wait for 3 min.' %
rawframe_path)
time.sleep(180)
waitmin += 3
apDisplay.printMsg('Waited for %d min so far' % waitmin)
if waitmin > self.waittime:
return False
return self.readImageFrame(rawframe_path, frame_number, offset,
crop_end, bin)
def insertImgAssessmentStatus(imgdata,
runname="run1",
assessment=None,
msg=True):
"""
Insert the assessment status
keep = True
reject = False
unassessed = None
"""
if assessment is True or assessment is False:
assessrun = appiondata.ApAssessmentRunData()
assessrun['session'] = imgdata['session']
#override to ALWAYS be 'run1'
#assessrun['name'] = runname
assessrun['name'] = "run1"
assessquery = appiondata.ApAssessmentData()
assessquery['image'] = imgdata
assessquery['assessmentrun'] = assessrun
assessquery['selectionkeep'] = assessment
assessquery.insert()
else:
apDisplay.printWarning("No image assessment made, invalid data: " +
str(assessment))
#check assessment
if msg is True:
finalassess = getImgAssessmentStatus(imgdata)
imgname = apDisplay.short(imgdata['filename'])
if finalassess is True:
astr = apDisplay.colorString("keep", "green")
elif finalassess is False:
astr = apDisplay.colorString("reject", "red")
elif finalassess is None:
astr = apDisplay.colorString("none", "yellow")
apDisplay.printMsg("Final image assessment: " + astr + " (" + imgname +
")")
return True
def commitToJensenDatabase(session_time, fulltomodata, stackdir, processdir,
stackname, description):
if not upload_to_Jensen_database:
return
raptordatabase = 0
apDisplay.printMsg("Uploding to Jensen tomography database")
alignrun = fulltomodata['alignrun']
tiltseries = fulltomodata['tiltseries']
nn = len(stackname)
mrcbase = processdir + '/align/' + stackname[0:-3]
mrcpath = mrcbase + '_full.rec'
if not os.path.exists(mrcpath):
mrcpath = mrcbase + '_part.rec'
if not os.path.exists(mrcpath):
mrcpath = ''
if mrcpath == '':
apDisplay.printWarning('No output full or part mrc found.')
raptordatabase = 1
else:
print '!!!0', ' mrcpath=', mrcpath
print '!!!1', ' stackdir=', stackdir, ' stackname=', stackname, ' processdir=', processdir, ' description=', description
imagelist = apTomo.getImageList([tiltseries])
defocus = imagelist[0]['preset']['defocus']
magnification = imagelist[0]['preset']['magnification']
q = appiondata.ApTiltsInAlignRunData(alignrun=alignrun)
r = q.query()
tomosettings = r[0]['settings']
tilt_min = tomosettings['tilt min']
tilt_max = tomosettings['tilt max']
tilt_step = tomosettings['tilt step']
dose = tomosettings['dose']
print '!!!2', ' defocus=', defocus, ' magnification=', magnification, ' tilt_min=', tilt_min, ' tilt_max=', tilt_max, ' tilt_step=', tilt_step, ' dose=', dose
#sessiondata = tiltdatalist[0]['session']
session_id = tiltseries.dbid
print '!!!3', ' session_id=', session_id, ' session_time=', session_time
return raptordatabase
def calculateEulerJumpsAndGoodBadParticles(self, uploadIterations):
''' calculate euler jumps for entire recon, currently based on EMAN ZXZ convention '''
if self.params['commit'] is True:
reconrunid = self.refinerunq.dbid
### make table entries of good-bad particles
apRecon.setGoodBadParticlesFromReconId(reconrunid)
### verify completed refinements and iterations
refinecomplete = self.verifyNumberOfCompletedRefinements(
multiModelRefinementRun=self.multiModelRefinementRun)
if self.multiModelRefinementRun is True:
### Euler jumpers calculated from ApMultiModelRefineRunData in multi-model case, looping over values in all single-model refinements
multimodelrunid = self.multimodelq.dbid
if len(uploadIterations) > 1:
apDisplay.printMsg(
"calculating euler jumpers for multi-model refinement="
+ str(multimodelrunid))
eulerjump = apEulerJump.ApEulerJump()
### TECHNICALLY, IT DOESN'T MAKE SENSE TO PASS THE RECONRUNID, SINCE PARTICLES CAN BE JUMPING,
### BUT AS FAR AS I CAN TELL, IT DOESN'T MAKE A DIFFERENCE IN THE RESULTING QUERY, SINCE THE VARIABLE
### IS ONLY USED TO QUERY FOR STACK PARTICLES, WHICH ARE IDENTICAL IN MULTI-MODEL AND SINGLE-MODEL REFINEMENT
### CASES. THEREFORE, THE LAST RECONRUNID IS PASSES ... * DMITRY
eulerjump.calculateEulerJumpsForEntireRecon(
reconrunid,
self.runparams['stackid'],
multimodelrunid=multimodelrunid)
else:
### Euler jumpers calculated from ApRefineRunData in single-model case
if len(uploadIterations) > 1 or \
(refinecomplete.itervalues().next()[-1] == self.runparams['numiter'] and len(refinecomplete.itervalues().next())>1):
apDisplay.printMsg("calculating euler jumpers for recon=" +
str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(
reconrunid, self.runparams['stackid'])
return
