def runCommand(self, cmd):
t0 = time.time()
if self.params['showcmd'] is True:
apDisplay.printColor("###################################", "magenta")
sys.stderr.write(
apDisplay.colorString("COMMAND: \n","magenta")
+apDisplay.colorString(cmd, "cyan")+"\n")
apDisplay.printColor("###################################", "cyan")
try:
if self.params['verbose'] is False:
logf = open('testsuite-programs.log' ,'a')
proc = subprocess.Popen(cmd, shell=True,
stdout=logf, stderr=logf)
else:
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
except:
apDisplay.printError("could not run command: "+cmd)
runtime = time.time() - t0
if self.params['showcmd'] is True:
apDisplay.printColor("###################################", "cyan")
apDisplay.printColor("command ran in "+apDisplay.timeString(runtime), "cyan")
if runtime < 1:
apDisplay.printError("command runtime was too short: "
+apDisplay.timeString(runtime))
elif runtime < 10:
apDisplay.printWarning("command runtime was very short: "
+apDisplay.timeString(runtime))
return False
if self.params['verbose'] is False:
logf.close()
return True
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.checkNoRefRun()
### convert stack to spider
spiderstack = self.createSpiderFile()
### create initialization template
if self.params['initmethod'] == 'allaverage':
templatefile = self.averageTemplate()
elif self.params['initmethod'] == 'selectrand':
templatefile = self.selectRandomParticles()
elif self.params['initmethod'] == 'randpart':
templatefile = self.pickRandomParticle()
elif self.params['initmethod'] == 'template':
templatefile = self.getTemplate()
else:
apDisplay.printError("unknown initialization method defined: "
+str(self.params['initmethod'])+" not in "+str(self.initmethods))
apDisplay.printColor("Running spider this can take awhile","cyan")
### run the alignment
aligntime = time.time()
pixrad = int(round(self.params['partrad']/self.stack['apix']/self.params['bin']))
alignedstack, self.partlist = alignment.refFreeAlignParticles(
spiderstack, templatefile,
self.params['numpart'], pixrad,
self.params['firstring'], self.params['lastring'],
rundir = ".")
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=False)
### convert stack to imagic
imagicstack = self.convertSpiderStack(alignedstack)
apFile.removeFile(alignedstack)
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = aligntime
self.insertNoRefRun(imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def runCommand(self, cmd):
t0 = time.time()
if self.params['showcmd'] is True:
apDisplay.printColor("###################################", "magenta")
sys.stderr.write(
apDisplay.colorString("COMMAND: \n","magenta")
+apDisplay.colorString(cmd, "cyan")+"\n")
apDisplay.printColor("###################################", "cyan")
try:
if self.params['verbose'] is False:
proc = subprocess.Popen(cmd, shell=True,
stdout=subprocess.PIPE)#, stderr=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
except:
apDisplay.printWarning("could not run command: "+cmd)
sys.exit(1)
return False
runtime = time.time() - t0
if self.params['showcmd'] is True:
apDisplay.printColor("###################################", "cyan")
apDisplay.printColor("command ran in "+apDisplay.timeString(runtime), "cyan")
if runtime < 10:
apDisplay.printWarning("command runtime was very short: "
+apDisplay.timeString(runtime))
sys.exit(1)
return False
#self.timestamp = apParam.makeTimestamp()
return True
def _printSummary(self):
"""
print summary statistics on last image
"""
### COP OUT
if self.params['background'] is True:
self.stats['count'] += 1
return
### THIS NEEDS TO BECOME MUCH MORE GENERAL, e.g. Peaks
tdiff = time.time()-self.stats['startseries']
if not self.params['continue'] or tdiff > 0.1:
count = self.stats['count']
#if(count != self.stats['lastcount']):
sys.stderr.write("\n\tSUMMARY: "+self.functionname+"\n")
self._printLine()
sys.stderr.write("\tTIME: \t"+apDisplay.timeString(tdiff)+"\n")
self.stats['timesum'] = self.stats['timesum'] + tdiff
self.stats['timesumsq'] = self.stats['timesumsq'] + (tdiff**2)
timesum = self.stats['timesum']
timesumsq = self.stats['timesumsq']
if(count > 1):
timeavg = float(timesum)/float(count)
timestdev = math.sqrt(float(count*timesumsq - timesum**2) / float(count*(count-1)))
timeremain = (float(timeavg)+float(timestdev))*self.stats['seriesleft']
sys.stderr.write("\tAVG TIME: \t"+apDisplay.timeString(timeavg,timestdev)+"\n")
#print "\t(- TOTAL:",apDisplay.timeString(timesum)," -)"
if(self.stats['seriesleft'] > 0):
sys.stderr.write("\t(- REMAINING TIME: "+apDisplay.timeString(timeremain)+" for "
+str(self.stats['seriesleft'])+" series -)\n")
#print "\tMEM: ",(mem.active()-startmem)/1024,"M (",(mem.active()-startmem)/(1024*count),"M)"
self.stats['count'] += 1
self._printLine()
def setVolumeMass(volumefile, apix=1.0, mass=1.0, rna=0.0):
"""
set the contour of 1.0 to the desired mass (in kDa) of the
macromolecule based on its density
use RNA to set the percentage of RNA in the structure
"""
if isValidVolume(volumefile) is False:
apDisplay.printError("Volume file is not valid")
procbin = apParam.getExecPath("proc2d")
emandir = os.path.dirname(procbin)
volumebin = os.path.join(emandir, "volume")
if not os.path.isfile(volumebin):
apDisplay.printWarning("failed to find volume program")
return False
command = "%s %s %.3f set=%.3f"%(
volumebin, volumefile, apix, mass
)
t0 = time.time()
proc = subprocess.Popen(command, shell=True)
proc.wait()
if time.time()-t0 < 0.01:
apDisplay.printWarning("failed to scale by mass in "+apDisplay.timeString(time.time()-t0))
return False
apDisplay.printMsg("finished scaling by mass in "+apDisplay.timeString(time.time()-t0))
return True
def applyEnvelopToDocFile(self, indocfile):
apDisplay.printMsg("Applying CTF to particles")
inf = open(indocfile, 'r')
outdocfile = os.path.splitext(indocfile)[0]+".envelop.lst"
outf = open(outdocfile, 'w')
cmdlist = []
partnum = 0
if self.params['pad'] is True:
scaleFactor = float(self.params['box']*self.params['padF']) / 4096.0
else:
scaleFactor = float(self.params['box']) / 4096.0
t0 = time.time()
for line in inf:
### get filename
partnum += 1
filename = line.strip().split()[0]
newfile = os.path.splitext(filename)[0]+".envelop.mrc"
self.applyEnvelope(filename, newfile, scaleFactor=scaleFactor, msg=False)
if not os.path.isfile(newfile):
apDisplay.printError("Ace 2 failed")
outf.write(newfile+"\t1\n")
numpart = partnum
inf.close()
outf.close()
timeper = (time.time()-t0)/float(numpart)
apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green")
apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green")
return outdocfile
def start(self):
self.runtime = 0
self.checkCoranRun()
### convert stack to spider
self.alignstackdata = self.getAlignedStack()
maskpixrad = self.params['maskrad']/self.alignstackdata['pixelsize']/self.params['bin']
boxpixdiam = int(math.ceil(maskpixrad)+1)*2
if boxpixdiam*self.params['bin'] > self.alignstackdata['boxsize']:
boxpixdiam = math.floor(self.alignstackdata['boxsize']/self.params['bin'])
clippixdiam = boxpixdiam*self.params['bin']
apDisplay.printMsg("Pixel mask radius="+str(maskpixrad))
oldalignedstack = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['imagicfile'])
alignedstackname = re.sub("\.", "_", self.alignstackdata['imagicfile'])+".spi"
alignedstack = os.path.join(self.params['rundir'], alignedstackname)
apFile.removeFile(alignedstack)
emancmd = ("proc2d %s %s spiderswap shrink=%d clip=%d,%d edgenorm"
%(oldalignedstack,alignedstack,self.params['bin'],clippixdiam,clippixdiam))
if self.params['numpart'] is not None:
emancmd += " last=%d"%(self.params['numpart']-1)
numpart = self.params['numpart']
else:
numpart = self.getNumAlignedParticles()
apEMAN.executeEmanCmd(emancmd, verbose=True)
### get number of processors
nproc = apParam.getNumProcessors()
esttime = classification.estimateTime(numpart, maskpixrad)
apDisplay.printColor("Running spider this can take awhile, estimated time: "+\
apDisplay.timeString(esttime),"cyan")
### do correspondence analysis
corantime = time.time()
self.contriblist = classification.correspondenceAnalysis( alignedstack,
boxsize=boxpixdiam, maskpixrad=maskpixrad,
numpart=numpart, numfactors=self.params['numfactors'], nproc=nproc)
corantime = time.time() - corantime
### make dendrogram
dendrotime = time.time()
classification.makeDendrogram(numfactors=min(3,self.params['numfactors']),nproc=nproc)
dendrotime = time.time() - dendrotime
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = corantime
self.insertCoranRun(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apFile.removeFile(alignedstack, warn=True)
apDisplay.printMsg("Correspondence Analysis time: "+apDisplay.timeString(corantime))
apDisplay.printMsg("Make Dendrogram time: "+apDisplay.timeString(dendrotime))
apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def alignParticlesToClasses(self, partStack, alignedClassStack, alignedPartStack):
t1 = time.time()
numPart = sparx.EMUtil.get_image_count(partStack)
# for some reason this reports more classes than exist
numClasses = sparx.EMUtil.get_image_count(alignedClassStack)
apDisplay.printMsg("aligning %d particles to %d classes"%(numPart, numClasses))
combinePartList = []
self.particleAlignData = {}
self.particleClass = {}
totalPart = 0
for newClassNum in range(numClasses):
genId, genClassNum = self.newClassToGenClass[newClassNum]
particleList = self.classMembersDict[genId][genClassNum]
apDisplay.printMsg("aligning %d particles to class %d of %d (gen %d, num %d)"
%(len(particleList), newClassNum, numClasses, genId, genClassNum))
partEMDataList = sparx.EMData.read_images(partStack, particleList, not self.headerOnly)
classEMData = sparx.get_im(alignedClassStack, newClassNum)
totalPart += len(particleList)
print newClassNum, particleList
for i in range(len(particleList)):
partEMData = partEMDataList[i]
partId = particleList[i]
self.particleClass[partId] = newClassNum
alignData = sparx.align2d(partEMData, classEMData,
self.xrange, self.yrange, self.transStep, self.firstRing,
self.lastRing, self.ringStep, self.mode)
self.particleAlignData[partId] = alignData
combinePartList.append(partId)
apDisplay.printColor("Finished aligning %d particles in %s"
%(totalPart, apDisplay.timeString(time.time() - t1)), "cyan")
t1 = time.time()
### write out complete alignment parameters for all generations & aligned stack
f = open("alignParticles.csv", "w")
count = 0
sys.stderr.write("writing %d aligned particles to file"%(len(combinePartList)))
self.origPartToAlignPartDict = {}
self.alignPartToOrigPartDict = {}
for partId in combinePartList:
self.origPartToAlignPartDict[partId] = count
self.alignPartToOrigPartDict[count] = partId
if count % 100 == 0:
sys.stderr.write(".")
# write alignments to file
alignData = self.particleAlignData[partId]
alpha, x, y, mirror, peak = alignData
f.write("%.3f\t%.3f\t%.3f\t%d\t%d\n" % (alpha, x, y, mirror, peak))
partEMData = sparx.EMData.read_images(partStack, [partId], not self.headerOnly)[0]
alignPartEMData = sparx.rot_shift2D(partEMData, alpha, x, y, mirror)
#we have to use count instead of partId, because not all images were aligned
alignPartEMData.write_image(alignedPartStack, count)
count += 1
f.close()
sys.stderr.write("\n")
apDisplay.printColor("Finished creating aligned stack of %d particles in %s"
%(count, apDisplay.timeString(time.time() - t1)), "cyan")
return
def removePtclsByJumps(self, particles, rejectlst):
eulerjump = apEulerJump.ApEulerJump()
numparts = len(particles)
apDisplay.printMsg("finding euler jumps for "+str(numparts)+" particles")
### check symmetry
symmetry = eulerjump.getSymmetry(self.params['reconid'], msg=True)
if not re.match("^[cd][0-9]+$", symmetry.lower()) and not re.match("^icos", symmetry.lower()):
apDisplay.printError("Cannot calculate euler jumps for symmetry: "+symmetry)
return
self.params['sym']=symmetry.lower()
### prepare file
f = open('jumps.txt','w', 0666)
f.write("#pnum\t")
headerlist = ('mean', 'median', 'stdev', 'min', 'max')
for key in headerlist:
f.write(key+"\t")
f.write("\n")
### get stack particles
stackparts = apStack.getStackParticlesFromId(self.params['stackid'])
### start loop
t0 = time.time()
medians = []
count = 0
apDisplay.printMsg("processing euler jumps for recon run="+str(self.params['reconid']))
for stackpart in stackparts:
count += 1
partnum = stackpart['particleNumber']
f.write('%d\t' % partnum)
jumpdata = eulerjump.getEulerJumpData(self.params['reconid'], stackpartid=stackpart.dbid, stackid=self.params['stackid'], sym=symmetry)
medians.append(jumpdata['median'])
if (jumpdata['median'] > self.params['avgjump']) and partnum not in rejectlst:
rejectlst.append(partnum)
for key in headerlist:
f.write("%.3f\t" % (jumpdata[key]))
f.write("\n")
if count % 1000 == 0:
timeremain = (time.time()-t0)/(count+1)*(numparts-count)
apDisplay.printMsg("particle=% 5d; median jump=% 3.2f, remain time= %s" % (partnum, jumpdata['median'],
apDisplay.timeString(timeremain)))
#f.flush()
### print stats
apDisplay.printMsg("-- median euler jumper stats --")
medians = numpy.asarray(medians, dtype=numpy.float32)
apDisplay.printMsg("mean/std :: "+str(round(medians.mean(),2))+" +/- "
+str(round(medians.std(),2)))
apDisplay.printMsg("min/max :: "+str(round(medians.min(),2))+" <> "
+str(round(medians.max(),2)))
perrej = round(100.0*float(numparts-len(rejectlst))/float(numparts),2)
apDisplay.printMsg("keeping "+str(numparts-len(rejectlst))+" of "+str(numparts)
+" particles ("+str(perrej)+"%) so far "
+" in "+apDisplay.timeString(time.time()-t0))
return rejectlst
def fillSimilarityMatrix(stackfile):
### Get initial correlation values
### this is really, really slow
numpart = apFile.numImagesInStack(stackfile)
similarfile = "similarities.dat"
if os.path.isfile(similarfile):
simf = open(similarfile, 'r')
simlist = []
count = 0
for line in simf:
count += 1
sline = line.strip()
slist = sline.split()
ccval = float(slist[2])
simlist.append(ccval)
simf.close()
apDisplay.printMsg("There are %d lines in the sim file: %s"%(count, similarfile))
if count == numpart*(numpart-1):
### we have a valid file already
return similarfile, simlist
### read data and estimate time
imagicdict = apImagicFile.readImagic(stackfile)
partarray = imagicdict['images']
numpart = partarray.shape[0]
boxsize = partarray.shape[1]
#timeper = 27.0e-9
timeper = 17.0e-9
apDisplay.printMsg("Computing CC values in about %s"
%(apDisplay.timeString(timeper*numpart**2*boxsize**2)))
### Computing CC values
simf = open(similarfile, 'w')
cctime = time.time()
simlist = []
for i in range(0, numpart):
if i % 100 == 99:
sys.stderr.write(".")
for j in range(i+1, numpart):
ccval = self.getCCValue(partarray[i],partarray[j])
str1 = "%05d %05d %.10f\n" % (i+1, j+1, ccval)
simf.write(str1)
str2 = "%05d %05d %.10f\n" % (j+1, i+1, ccval)
simf.write(str2)
simlist.append(ccval)
sys.stderr.write("\n")
simf.close()
del partarray
del imagicdict['images']
apDisplay.printMsg("CC calc time: %s :: %s per part :: %s per part per pixel"
%(apDisplay.timeString(time.time()-cctime),
apDisplay.timeString((time.time()-cctime)/numpart**2),
apDisplay.timeString((time.time()-cctime)/numpart**2/boxsize**2)))
return similarfile, simlist
def applyEnvelopeAndCTF(self, stack):
### get defocus lists
numpart = self.params['projcount']
cut = int(numpart/80.0)+1
apDisplay.printMsg("%d particles per dot"%(cut))
if len(self.deflist1) == 0:
self.getListOfDefoci(numpart)
### break up particles
partlistdocfile = apXmipp.breakupStackIntoSingleFiles(stack, filetype="mrc")
t0 = time.time()
apDisplay.printMsg("Applying CTF and Envelop to particles")
### apply CTF using ACE2
ctfapplydocfile = self.applyCTFToDocFile(partlistdocfile)
### apply Envelop using ACE2
envelopdocfile = self.applyEnvelopToDocFile(ctfapplydocfile)
### correct CTF using ACE2
if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True:
ctfcorrectdocfile = self.correctCTFToDocFile(envelopdocfile)
else:
ctfcorrectdocfile = envelopdocfile
timeper = (time.time()-t0)/float(numpart)
apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green")
apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green")
### write corrected particle list to doc file
ctfpartlist = []
ctfpartlistfile = os.path.join(self.params['rundir'], "ctfpartlist.lst")
inf = open(ctfcorrectdocfile, 'r')
outf = open(ctfpartlistfile, "w")
for line in inf:
### get filename
filename = line.strip().split()[0]
if not os.path.isfile(filename):
apDisplay.printError("CTF and envelop apply failed")
ctfpartlist.append(filename)
outf.write(filename+"\t1\n")
inf.close()
outf.close()
### merge individual files into a common stack
ctfstack = os.path.join(self.params['rundir'], "ctfstack.hed")
apXmipp.gatherSingleFilesIntoStack(ctfpartlistfile, ctfstack, filetype="mrc")
if self.params['pad'] is True:
emancmd = "proc2d %s %s.clip.hed clip=%d,%d" % (ctfstack, ctfstack[:-4], self.params['box'], self.params['box'])
apParam.runCmd(emancmd, "EMAN")
shutil.move("%s.clip.hed" % ctfstack[:-4], "%s.hed" % ctfstack[:-4])
shutil.move("%s.clip.img" % ctfstack[:-4], "%s.img" % ctfstack[:-4])
return ctfstack, ctfpartlist
def projMatchRefine(self, classnum, volfile, alignstack, eulerfile, boxsize, numpart, pixrad, iternum):
APSHout = backproject.alignAPSH(volfile, alignstack, eulerfile, classnum, boxsize, numpart, pixrad, self.timestamp, iternum)
### check APSH output
if (os.path.isfile(APSHout) is False):
apDisplay.printError("AP SH alignment did not generate a valid output file. Please check parameters and rerun!")
apsh = open(APSHout, "r")
neweulerdoc = os.path.join(self.params['rundir'], str(classnum),"newEulersdoc-%03d.spi"%(iternum))
neweulerfile = open(neweulerdoc, "w")
rotshiftdoc = os.path.join(self.params['rundir'], str(classnum),"rotShiftdoc-%03d.spi"%(iternum))
rotshiftfile = open(rotshiftdoc, "w")
starttime = time.time()
count = 0
for line in apsh.readlines():
value = line.split()
try:
int(value[0])
except:
#apDisplay.printMsg(line)
continue
key = int(float(value[6]))
rot = float(value[7])
cumX = float(value[14]) #float(value[8])
cumY = float(value[15]) #float(value[9])
psi = float(value[2])
theta = float(value[3])
phi = float(value[4])
mirror = int(float(value[16]))
### rotate and shift particle
APSHstack = backproject.rotshiftParticle(alignstack, key, rot, cumX, cumY, mirror, iternum, self.timestamp, str(classnum))
### write out new euler file
eulerline = operations.spiderOutLine(key, [psi, theta, phi])
neweulerfile.write(eulerline)
rotshiftline = operations.spiderOutLine(key, [rot, 1.00, cumX, cumY])
rotshiftfile.write(rotshiftline)
count+=1
if (count%20) == 0:
apDisplay.printColor(str(numpart-count)+" particles left", "cyan")
apDisplay.printColor("Estimated time left is "+apDisplay.timeString(((time.time()-starttime)/count)*(numpart-count)), "cyan")
apDisplay.printColor("finished rotating and shifting particles "+apDisplay.timeString(time.time()-starttime), "cyan")
neweulerfile.close()
rotshiftfile.close()
return APSHout, APSHstack, neweulerdoc
def start(self):
"""
This is the core of your function.
You decide what happens here!
"""
### try and get the appion instruments
self.getAppionInstruments()
### create new session, so we have a place to store the log file
self.createNewSession()
mrclist = self.getImagesInDirectory(self.params["imagedir"])
for i in range(min(len(mrclist), 6)):
print mrclist[i]
numinseries = 1
seriescount = 1
count = 1
t0 = time.time()
for mrcfile in mrclist:
### rename image
newimagepath = self.newImagePath(mrcfile, numinseries)
### get image dimensions
dims = self.getImageDimensions(mrcfile)
### set pixel size in database
self.updatePixelSizeCalibration()
## read the file. images should be small enough to read into memory
imagearray = self.readFile(mrcfile)
### upload image
self.uploadImageInformation(imagearray, newimagepath, dims, seriescount, numinseries)
### counting
numinseries += 1
if numinseries % (self.params["seriessize"] + 1) == 0:
### reset series counter
seriescount += 1
numinseries = 1
# print count, seriescount, numinseries
timeperimage = (time.time() - t0) / float(count)
apDisplay.printMsg("time per image: %s" % (apDisplay.timeString(timeperimage)))
esttime = timeperimage * (len(mrclist) - count)
apDisplay.printMsg(
"estimated time remaining for %d of %d images: %s"
% (len(mrclist) - count, len(mrclist), apDisplay.timeString(esttime))
)
### counting
count += 1
def start(self):
### check for existing run
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
selectrundata = selectrunq.query(readimages=False)
if selectrundata:
apDisplay.printError("Runname already exists")
### stack data
stackdata = apStack.getOnlyStackData(self.params['stackid'])
### stack particles
stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
stackparts.reverse()
### selection run for first particle
oldselectrun = stackparts[0]['particle']['selectionrun']
### set selection run
manualparamsq = appiondata.ApManualParamsData()
manualparamsq['diam'] = self.getDiamFromSelectionRun(oldselectrun)
manualparamsq['oldselectionrun'] = oldselectrun
manualparamsq['trace'] = False
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['hidden'] = False
selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
selectrunq['session'] = apStack.getSessionDataFromStackId(self.params['stackid'])
selectrunq['manparams'] = manualparamsq
### insert particles
apDisplay.printMsg("Inserting particles into database")
count = 0
t0 = time.time()
startmem = mem.active()
numpart = len(stackparts)
for stackpart in stackparts:
count += 1
if count > 10 and count%100 == 0:
perpart = (time.time()-t0)/float(count+1)
apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"%
(count, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart),
apDisplay.timeString(perpart*(numpart-count))), "blue")
oldpartdata = stackpart['particle']
newpartq = appiondata.ApParticleData(initializer=oldpartdata)
newpartq['selectionrun'] = selectrunq
if self.params['commit'] is True:
newpartq.insert()
apDisplay.printMsg("Completed in %s"%(apDisplay.timeString(time.time()-t0)))
def calcResolution(self, partlist, stackfile, apix):
### group particles by refnum
reflistsdict = {}
for partdict in partlist:
refnum = partdict['template']
partnum = partdict['num']
if not refnum in reflistsdict:
reflistsdict[refnum] = []
reflistsdict[refnum].append(partnum)
### get resolution
self.resdict = {}
boxsizetuple = apFile.getBoxSize(stackfile)
boxsize = boxsizetuple[0]
for refnum in reflistsdict.keys():
partlist = reflistsdict[refnum]
esttime = 3e-6 * len(partlist) * boxsize**2
apDisplay.printMsg("Ref num %d; %d parts; est time %s"
%(refnum, len(partlist), apDisplay.timeString(esttime)))
frcdata = apFourier.spectralSNRStack(stackfile, apix, partlist, msg=False)
frcfile = "frcplot-%03d.dat"%(refnum)
apFourier.writeFrcPlot(frcfile, frcdata, apix, boxsize)
res = apFourier.getResolution(frcdata, apix, boxsize)
self.resdict[refnum] = res
return
def convertSQLtoEulerTree(self, results):
t0 = time.time()
eulertree = []
for row in results:
if len(row) < 11:
apDisplay.printError("delete MySQL cache file and run again")
try:
eulerpair = { 'part1': {}, 'part2': {} }
eulerpair['part1']['partid'] = int(row[0])
eulerpair['part1']['dbid'] = int(row[1])
eulerpair['part1']['euler1'] = float(row[2])
eulerpair['part1']['euler2'] = float(row[3])
eulerpair['part1']['euler3'] = float(row[4])
eulerpair['part1']['mirror'] = self.nullOrValue(row[5])
eulerpair['part1']['reject'] = not self.nullOrValue(row[6])
eulerpair['part1']['tilt'] = apStack.getStackParticleTilt(eulerpair['part1']['dbid'])
eulerpair['part2']['partid'] = int(row[7])
eulerpair['part2']['dbid'] = int(row[8])
eulerpair['part2']['euler1'] = float(row[9])
eulerpair['part2']['euler2'] = float(row[10])
eulerpair['part2']['euler3'] = float(row[11])
eulerpair['part2']['mirror'] = self.nullOrValue(row[12])
eulerpair['part2']['reject'] = not self.nullOrValue(row[13])
eulerpair['part2']['tilt'] = apStack.getStackParticleTilt(eulerpair['part2']['dbid'])
eulertree.append(eulerpair)
except:
print row
apDisplay.printError("bad row entry")
apDisplay.printMsg("Converted "+str(len(eulertree))+" eulers in "+apDisplay.timeString(time.time()-t0))
return eulertree
def getNumAlignedParticles(self):
t0 = time.time()
self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
oldalignedstack = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['imagicfile'])
numpart = apFile.numImagesInStack(oldalignedstack)
apDisplay.printMsg("numpart="+str(numpart)+" in "+apDisplay.timeString(time.time()-t0))
return numpart
def xmippNormStack(self, inStackPath, outStackPath):
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath)
### setup Xmipp command
xmippexe = apParam.getExecPath("xmipp_normalize", die=True)
apDisplay.printMsg("Using Xmipp to normalize particle stack")
normtime = time.time()
xmippopts = ( " "
+" -i %s"%os.path.join(self.params['rundir'],selfile)
+" -method Ramp "
+" -background circle %i"%(self.stack['boxsize']/self.params['bin']*0.4)
+" -remove_black_dust"
+" -remove_white_dust"
+" -thr_black_dust -%.2f"%(self.params['xmipp-norm'])
+" -thr_white_dust %.2f"%(self.params['xmipp-norm'])
)
xmippcmd = xmippexe+" "+xmippopts
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
normtime = time.time() - normtime
apDisplay.printMsg("Xmipp normalization time: "+apDisplay.timeString(normtime))
### recombine particles to a single imagic stack
tmpstack = "tmp.xmippStack.hed"
apXmipp.gatherSingleFilesIntoStack(selfile,tmpstack)
apFile.moveStack(tmpstack,outStackPath)
### clean up directory
apFile.removeFile(selfile)
apFile.removeDir("partfiles")
def printTimeStats(self, name, timelist):
if len(timelist) < 2:
return
meantime = self.stats['timesum']/float(self.stats['count'])
timearray = numpy.array(timelist, dtype=numpy.float64)
apDisplay.printColor("%s: %s (%.2f percent)"%
(name, apDisplay.timeString(timearray.mean(), timearray.std()), 100*timearray.mean()/meantime), "blue")
def importPicks(self, picks1, picks2, tight=False, msg=True):
t0 = time.time()
#print picks1
#print self.currentpicks1
curpicks1 = numpy.asarray(self.currentpicks1)
curpicks2 = numpy.asarray(self.currentpicks2)
#print curpicks1
# get picks
apTiltTransform.setPointsFromArrays(curpicks1, curpicks2, self.data)
pixdiam = self.data['pixdiam']
if tight is True:
pixdiam /= 4.0
#print self.data, pixdiam
list1, list2 = apTiltTransform.alignPicks2(picks1, picks2, self.data, limit=pixdiam, msg=msg)
if list1.shape[0] == 0 or list2.shape[0] == 0:
apDisplay.printWarning("No new picks were found")
# merge picks
newpicks1, newpicks2 = apTiltTransform.betterMergePicks(curpicks1, list1, curpicks2, list2, msg=msg)
newparts = newpicks1.shape[0] - curpicks1.shape[0]
# copy over picks
self.currentpicks1 = newpicks1
self.currentpicks2 = newpicks2
if msg is True:
apDisplay.printMsg("Inserted "+str(newparts)+" new particles in "+apDisplay.timeString(time.time()-t0))
return True
def createSpiderFile(self):
"""
takes the stack file and creates a spider file ready for processing
"""
emancmd = "proc2d "
if not os.path.isfile(self.stack['file']):
apDisplay.printError("stackfile does not exist: "+self.stack['file'])
emancmd += self.stack['file']+" "
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "apix="+str(self.stack['apix'])+" "
if self.params['lowpass'] > 0:
emancmd += "lp="+str(self.params['lowpass'])+" "
if self.params['highpass'] > 0:
emancmd += "hp="+str(self.params['highpass'])+" "
if self.params['bin'] > 1:
clipboxsize = self.boxsize*self.params['bin']
emancmd += "shrink="+str(self.params['bin'])+" "
emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" "
emancmd += "last="+str(self.params['numpart']-1)+" "
emancmd += "spider edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
return spiderstack
def getTransformImageIds(transformdata):
t0 = time.time()
img1 = transformdata.special_getitem('image1', dereference=False).dbid
img2 = transformdata.special_getitem('image2', dereference=False).dbid
if time.time()-t0 > 0.3:
apDisplay.printMsg("long image query "+apDisplay.timeString(time.time()-t0))
return img1, img2
def optimizeAngles(self, msg=True):
t0 = time.time()
### run find theta
na1 = numpy.array(self.currentpicks1, dtype=numpy.int32)
na2 = numpy.array(self.currentpicks2, dtype=numpy.int32)
fittheta = radermacher.tiltang(na1, na2)
if not fittheta or not 'wtheta' in fittheta:
return
theta = fittheta['wtheta']
thetadev = fittheta['wthetadev']
if msg is True:
thetastr = ("%3.3f +/- %2.2f" % (theta, thetadev))
tristr = apDisplay.orderOfMag(fittheta['numtri'])+" of "+apDisplay.orderOfMag(fittheta['tottri'])
tristr = (" (%3.1f " % (100.0 * fittheta['numtri'] / float(fittheta['tottri'])))+"%) "
apDisplay.printMsg("Tilt angle "+thetastr+tristr)
self.data['theta'] = fittheta['wtheta']
### run optimize angles
lastiter = [80,80,80]
count = 0
totaliter = 0
while max(lastiter) > 75 and count < 30:
count += 1
lsfit = self.runLeastSquares()
lastiter[2] = lastiter[1]
lastiter[1] = lastiter[0]
lastiter[0] = lsfit['iter']
totaliter += lsfit['iter']
if msg is True:
apDisplay.printMsg("Least Squares: "+str(count)+" rounds, "+str(totaliter)
+" iters, rmsd of "+str(round(lsfit['rmsd'],4))+" pixels in "+apDisplay.timeString(time.time()-t0))
return
def convertStackToSpider(self, emanstackfile, classnum):
"""
takes the stack file and creates a spider file ready for processing
"""
if not os.path.isfile(emanstackfile):
apDisplay.printError("stackfile does not exist: "+emanstackfile)
### first high pass filter particles
apDisplay.printMsg("pre-filtering particles")
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
emancmd = ("proc2d "+emanstackfile+" "+emanstackfile
+" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
+" inplace")
apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc2d "
emancmd += emanstackfile+" "
spiderstack = os.path.join(self.params['rundir'], str(classnum), "otrstack"+self.timestamp+".spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
return spiderstack
def getTiltTransformFromParticle(partdata):
t0 = time.time()
### figure out if its particle 1 or 2
tiltpartq1 = appiondata.ApTiltParticlePairData()
tiltpartq1['particle1'] = partdata
tiltpartdatas1 = tiltpartq1.query(results=1, readimages=False)
tiltpartq2 = appiondata.ApTiltParticlePairData()
tiltpartq2['particle2'] = partdata
tiltpartdatas2 = tiltpartq2.query(results=1, readimages=False)
if tiltpartdatas1 and not tiltpartdatas2:
imgnum = 1
transformdata = tiltpartdatas1[0]['transform']
otherpartdata = tiltpartdatas1[0]['particle2']
elif not tiltpartdatas1 and tiltpartdatas2:
imgnum = 2
transformdata = tiltpartdatas2[0]['transform']
otherpartdata = tiltpartdatas2[0]['particle1']
else:
print partdata
print tiltpartdatas1
print tiltpartdatas2
apDisplay.printError("failed to get tilt pair data")
if time.time()-t0 > 1.0:
apDisplay.printMsg("long getTiltTransFromPart1 "+apDisplay.timeString(time.time()-t0))
return imgnum, transformdata, otherpartdata
def getParticleTiltRotationAnglesOTR(stackpartdata):
partdata = stackpartdata['particle']
imgnum, transformdata, otherpartdata = getTiltTransformFromParticle(partdata)
t0 = time.time()
tiltangle1, tiltangle2 = apDatabase.getTiltAnglesDegFromTransform(transformdata)
if time.time()-t0 > 1.0:
apDisplay.printMsg("long angle query "+apDisplay.timeString(time.time()-t0))
if imgnum == 1:
### negative case, tilt picker theta < 0
tiltrot = transformdata['image1_rotation']
theta = transformdata['tilt_angle']
notrot = transformdata['image2_rotation']
tiltangle = tiltangle1 - tiltangle2
elif imgnum == 2:
### positive case, tilt picker theta > 0
tiltrot = transformdata['image2_rotation']
theta = transformdata['tilt_angle']
notrot = transformdata['image1_rotation']
tiltangle = tiltangle2 - tiltangle1
else:
#no particle pair info was found or some other problem
print partdata
apDisplay.printError("failed to get tilt pair data or some other problem")
if transformdata.timestamp < datetime.datetime(2009, 2, 19, 0, 0, 0):
### bugfix for switched tilt axis angles, before Feb 19, 2009
#apDisplay.printWarning("Switching angles")
temprot = notrot
notrot = tiltrot
tiltrot = temprot
#print "tr=%.2f, th=%.2f, nr=%.2f, tilt=%.2f"%(tiltrot, theta, notrot, tiltangle)
return tiltrot, theta, notrot, tiltangle
def rctParticleShift(volfile, origstackfile, eulerdocfile, iternum, numpart, pixrad, timestamp, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
starttime = time.time()
### create corresponding projections
projstackfile = "projstack%s-%03d.spi"%(timestamp, iternum)
projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext)
### clean up files
ccdocfile = "ccdocfile%s-%03d.spi"%(timestamp, iternum)
apFile.removeFile(ccdocfile)
alignstackfile = "alignstack%s-%03d.spi"%(timestamp, iternum)
apFile.removeFile(alignstackfile)
### align particles to projection
apDisplay.printMsg("Shifting particles")
crossCorrelateAndShift(origstackfile, projstackfile, alignstackfile, ccdocfile, numpart)
if not os.path.isfile(alignstackfile):
apDisplay.printError("aligned stack file not found: "+alignstackfile)
apDisplay.printColor("finished correlations in "+apDisplay.timeString(time.time()-starttime), "cyan")
return alignstackfile
def backprojectCG(stackfile, eulerdocfile, volfile, numpart, pixrad, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
### setup
starttime = time.time()
stackfile = spyder.fileFilter(stackfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(stackfile+dataext):
apDisplay.printError("stack file not found: "+stackfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(volfile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("BP CG",
stackfile+"@*****", #stack file
"1-%d"%(numpart), #number of particles
str(pixrad), #particle radius
eulerdocfile, #angle doc file
"N", #has symmetry?, does not work
volfile, #filename for volume
"%.1e,%.1f" % (1.0e-5, 0.0), #error, chi^2 limits
"%d,%d" % (25,1), #iterations, 1st derivative mode
"2000", #lambda - higher=less sensitive to noise
)
mySpider.close()
apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext=".spi"):
"""
project 3D volumes using given Euler angles
"""
starttime = time.time()
volfile = spyder.fileFilter(volfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
projstackfile = spyder.fileFilter(projstackfile)
if not os.path.isfile(volfile+dataext):
apDisplay.printError("volume file not found: "+volfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(projstackfile)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("PJ 3Q",
volfile, #input vol file
str(pixrad), #pixel radius
"1-%d"%(numpart), #number of particles
eulerdocfile, #Euler DOC file
projstackfile+"@*****", #output projections
)
mySpider.close()
apDisplay.printColor("finished projections in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def getOverlap(self, image1, image2, msg=True):
t0 = time.time()
bestOverlap, tiltOverlap = apTiltTransform.getOverlapPercent(image1, image2, self.data)
overlapStr = str(round(100*bestOverlap,2))+"% and "+str(round(100*tiltOverlap,2))+"%"
if msg is True:
apDisplay.printMsg("Found overlaps of "+overlapStr+" in "+apDisplay.timeString(time.time()-t0))
self.data['overlap'] = bestOverlap
def processTiltPair(self,
imgfile1,
imgfile2,
picks1,
picks2,
tiltangle,
outfile,
pixdiam=20.0,
tiltaxis=-7.0,
msg=True):
"""
Inputs:
imgfile1
imgfile2
picks1, 2xN numpy array
picks2, 2xN numpy array
tiltangle
outfile
Modifies:
outfile
Output:
None, failed
True, success
"""
### pre-load particle picks
if len(picks1) < 10 or len(picks2) < 10:
if msg is True:
apDisplay.printWarning(
"Not enough particles ot run program on image pair")
return None
### setup tilt data
self.data['theta'] = tiltangle
self.data['shiftx'] = 0.0
self.data['shifty'] = 0.0
self.data['gamma'] = tiltaxis
self.data['phi'] = tiltaxis
self.data['scale'] = 1.0
self.data['pixdiam'] = pixdiam
### open image file 1
img1 = self.openImageFile(imgfile1)
if img1 is None:
apDisplay.printWarning("Could not read image: " + imgfile1)
return None
### open tilt file 2
img2 = self.openImageFile(imgfile2)
if img1 is None:
apDisplay.printWarning("Could not read image: " + imgfile1)
return None
### guess the shift
t0 = time.time()
if msg is True:
apDisplay.printMsg("Refining tilt axis angles")
origin, newpart, snr, bestang = apTiltShift.getTiltedCoordinates(
img1, img2, tiltangle, picks1, True, tiltaxis, msg=msg)
self.data['gamma'] = float(bestang)
self.data['phi'] = float(bestang)
if snr < 2.0:
if msg is True:
apDisplay.printWarning("Low confidence in initial shift")
return None
self.currentpicks1 = [origin]
self.currentpicks2 = [newpart]
### search for the correct particles
self.importPicks(picks1, picks2, tight=False, msg=msg)
if len(self.currentpicks1) < 4:
apDisplay.printWarning("Failed to find any particle matches")
return None
self.deleteFirstPick()
self.printData(msg)
for i in range(4):
self.clearBadPicks(msg)
if len(self.currentpicks1) < 5 or len(self.currentpicks2) < 5:
if msg is True:
apDisplay.printWarning(
"Not enough particles to optimize angles")
return None
self.optimizeAngles(msg)
self.printData(msg)
self.clearBadPicks(msg)
self.clearBadPicks(msg)
if len(self.currentpicks1) < 5 or len(self.currentpicks2) < 5:
if msg is True:
apDisplay.printWarning(
"Not enough particles to optimize angles")
return None
self.optimizeAngles(msg)
self.printData(msg)
self.clearBadPicks(msg)
self.importPicks(picks1, picks2, tight=False, msg=msg)
self.clearBadPicks(msg)
self.printData(msg)
if len(self.currentpicks1) < 5 or len(self.currentpicks2) < 5:
if msg is True:
apDisplay.printWarning(
"Not enough particles to optimize angles")
return None
self.optimizeAngles(msg)
self.printData(msg)
self.getOverlap(img1, img2, msg)
if msg is True:
apDisplay.printMsg("Completed alignment of " +
str(len(self.currentpicks1)) +
" particle pairs in " +
apDisplay.timeString(time.time() - t0))
self.saveData(imgfile1, imgfile2, outfile)
self.printData(msg)
return True
def refBasedAlignParticles(stackfile,
templatestack,
origstackfile,
xysearch,
xystep,
numpart,
numtemplate,
firstring=2,
lastring=100,
dataext=".spi",
iternum=1,
oldpartlist=None):
"""
inputs:
stack
template
search params
outputs:
aligned stack
rotation/shift params
"""
### setup
if dataext in templatestack:
templatestack = templatestack[:-4]
if dataext in stackfile:
stackfile = stackfile[:-4]
if dataext in origstackfile:
origstackfile = origstackfile[:-4]
t0 = time.time()
rundir = "alignments"
apParam.createDirectory(rundir)
nproc = apParam.getNumProcessors()
### remove previous iterations
apFile.removeFile(rundir + "/paramdoc%02d%s" % (iternum, dataext))
### perform alignment, should I use 'AP SH' instead?
mySpider = spyder.SpiderSession(dataext=dataext,
logo=True,
nproc=nproc,
log=False)
mySpider.toSpider(
"AP MQ",
spyder.fileFilter(templatestack) + "@**", # reference image series
"1-" + str(numtemplate), # enter number of templates of doc file
str(int(xysearch)) + "," +
str(int(xystep)), # translation search range, step size
str(int(firstring)) + "," +
str(int(lastring)), # first and last ring for rotational correlation
spyder.fileFilter(stackfile) + "@******", # unaligned image series
"1-" + str(numpart), # enter number of particles of doc file
rundir + ("/paramdoc%02d" % (iternum)), # output angles document file
)
mySpider.close()
"""
### perform alignment, should I use 'AP SH' instead?
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("AP SH",
spyder.fileFilter(templatestack)+"@**", # reference image series
"1-"+str(numtemplate), # enter number of templates of doc file
str(int(xysearch))+","+str(int(xystep)), # translation search range, step size
str(int(firstring))+","+str(int(lastring)), # first and last ring for rotational correlation
"*", # initial angles
spyder.fileFilter(stackfile)+"@******", # unaligned image series
"1-"+str(numpart), # enter number of particles of doc file
"*", # initial angles
"0.0", # angle search
"1", # check mirrors and shift/rotate input
rundir+("/paramdoc%02d" % (iternum)), # output angles document file
)
mySpider.close()
"""
### convert spider rotation, shift data to python
docfile = rundir + ("/paramdoc%02d" % (iternum)) + dataext
picklefile = rundir + ("/paramdoc%02d" % (iternum)) + ".pickle"
if oldpartlist is not None and iternum > 1:
apDisplay.printMsg("updating particle doc info")
partlist = updateRefBasedDocFile(oldpartlist, docfile, picklefile)
elif iternum == 1:
apDisplay.printMsg("reading initial particle doc info")
partlist = readRefBasedDocFile(docfile, picklefile)
else:
apDisplay.printError(
"reading (not updating) particle doc info on iteration " +
str(iternum))
### write aligned stack -- with python loop
alignedstack = rundir + ("/alignedstack%02d" % (iternum))
alignStack(origstackfile, alignedstack, partlist, dataext)
### average stack
emancmd = ("proc2d " + alignedstack + dataext + " " + rundir +
("/avgimg%02d" % (iternum)) + ".mrc " + " average")
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
td1 = time.time() - t0
apDisplay.printMsg("completed alignment of " + str(numpart) +
" particles in " + apDisplay.timeString(td1))
if numpart < 1:
apDisplay.printError("Failed to find any particles")
return alignedstack + dataext, partlist
def refFreeAlignParticles(stackfile,
template,
numpart,
pixrad,
firstring=2,
lastring=100,
dataext=".spi",
rundir="alignment"):
"""
inputs:
stack
template
search params
outputs:
aligned stack
rotation/shift params
"""
### setup
if dataext in template:
template = template[:-4]
if dataext in stackfile:
stackfile = stackfile[:-4]
t0 = time.time()
apParam.createDirectory(rundir)
### remove previous iterations
numiter = 0
while os.path.isfile(rundir + "/avgimg%02d%s" % (numiter + 1, dataext)):
apFile.removeFile(rundir + "/avgimg%02d%s" % (numiter + 1, dataext))
pngfile = rundir + "/avgimg%02d%s" % (numiter + 1, ".png")
apFile.removeFile(pngfile)
numiter += 1
### perform alignment
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
apDisplay.printMsg("Performing particle alignment")
# copy template to memory
mySpider.toSpiderQuiet("CP", (template + "@1"), "_9")
mySpider.toSpider("AP SR",
spyder.fileFilter(stackfile) + "@******",
"1-" + str(numpart), str(int(pixrad)),
str(int(firstring)) + "," + str(int(lastring)), "_9",
rundir + "/avgimg**", rundir + "/paramdoc**")
mySpider.close()
### find number of iterations
numiter = 0
while os.path.isfile(rundir + "/avgimg%02d%s" % (numiter + 1, dataext)):
emancmd = ("proc2d " + " " + rundir + "/avgimg" +
("%02d%s" % (numiter + 1, dataext)) + " " + rundir +
"/avgimg" + ("%02d%s" % (numiter + 1, ".png")))
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
numiter += 1
if numiter == 0:
apDisplay.printError("alignment failed, no iterations were found")
emancmd = ("proc2d " + " " + rundir + "/avgimg" +
("%02d%s" % (numiter, dataext)) + " " + rundir + "/average.mrc")
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apDisplay.printMsg(
str(numiter) + " alignment iterations were run by spider")
### convert spider rotation, shift data to python
docfile = rundir + ("/paramdoc%02d" % (numiter)) + dataext
picklefile = rundir + ("/paramdoc%02d" % (numiter)) + ".pickle"
partlist = readRefFreeDocFile(docfile, picklefile)
### write aligned stack -- with python loop
alignedstack = "alignedstack"
alignStack(stackfile, alignedstack, partlist, dataext)
td1 = time.time() - t0
apDisplay.printMsg("completed alignment of " + str(numpart) +
" particles in " + apDisplay.timeString(td1))
return ("alignedstack.spi", partlist)
def alignStack(oldstack, alignedstack, partlist, dataext=".spi"):
"""
write aligned stack -- with python loop
inputs:
oldstack
newstack (empty)
list of particle dictionaries for operations
modifies:
newstack
output:
none
I tried this loop in both spider and python;
python was faster?!? -neil
"""
if not os.path.isfile(oldstack + dataext):
apDisplay.printError("Could not find original stack: " + oldstack +
dataext)
boxsize = apFile.getBoxSize(oldstack + dataext)
apDisplay.printMsg("applying alignment parameters to stack")
apFile.removeFile(alignedstack + dataext)
count = 0
t0 = time.time()
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext,
logo=True,
nproc=nproc,
log=False)
#create stack in core
numpart = len(partlist)
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d" % (boxsize), #boxsize
str(numpart + 1), #num part to create in memory
str(numpart + 1), #max particle number
)
for partdict in partlist:
partnum = partdict['num']
#if partdict['num'] in [3,6,7]:
# print partdict['num'], partdict['template'], partdict['mirror'], round(partdict['rot'],3)
### Rotate and Shift operations
count += 1
#rotate/shift
mySpider.toSpiderQuiet(
"RT SQ",
spyder.fileFilter(oldstack) + "@" + ("%06d" % (partnum)),
"_1",
str(partdict['rot']),
str(partdict['xshift']) + "," + str(partdict['yshift']),
)
#mirror, if necessary
if 'mirror' in partdict and partdict['mirror'] is True:
mySpider.toSpiderQuiet(
"MR",
"_1",
"_2@" + ("%06d" % (partnum)),
"Y",
)
else:
mySpider.toSpiderQuiet(
"CP",
"_1",
"_2@" + ("%06d" % (partnum)),
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP",
"_2@",
spyder.fileFilter(alignedstack) + "@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE",
"_2",
)
mySpider.close()
apDisplay.printMsg("Completed transforming %d particles in %s" %
(count, apDisplay.timeString(time.time() - t0)))
if count < 1:
apDisplay.printError("Failed to transform any particles")
if not os.path.isfile(alignedstack + dataext):
apDisplay.printError("Failed to create stack " + alignedstack +
dataext)
return
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### test insert to make sure data is not overwritten
self.params['runtime'] = 0
#self.checkDuplicateRefBasedRun()
### set box size
self.boxsize = int(
math.floor(self.stack['boxsize'] / self.params['bin'] / 2.0)) * 2
### convert stack to spider
spiderstack = self.createSpiderFile()
### create template stack
templatestack = self.createTemplateStack()
### run the alignment
aligntime = time.time()
usestack = spiderstack
oldpartlist = None
for i in range(self.params['numiter']):
iternum = i + 1
apDisplay.printColor("\n\nITERATION " + str(iternum), "green")
alignedstack, partlist = alignment.refBasedAlignParticles(
usestack,
templatestack,
spiderstack,
self.params['xysearch'],
self.params['xystep'],
self.params['numpart'],
self.params['numtemplate'],
self.params['firstring'],
self.params['lastring'],
iternum=iternum,
oldpartlist=oldpartlist)
oldpartlist = partlist
usestack = alignedstack
templatestack = self.updateTemplateStack(alignedstack, partlist,
iternum)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: " + spiderstack)
apFile.removeFile(spiderstack, warn=True)
### convert aligned stack to imagic
finalspistack = "aligned.spi"
shutil.move(alignedstack, finalspistack)
imagicstack = "aligned.hed"
apFile.removeStack(imagicstack)
emancmd = "proc2d " + finalspistack + " " + imagicstack
apEMAN.executeEmanCmd(emancmd, verbose=True)
### average stack
apStack.averageStack(imagicstack)
### calculate resolution for each reference
apix = self.stack['apix'] * self.params['bin']
self.calcResolution(partlist, imagicstack, apix)
if self.params['commit'] is True:
apDisplay.printMsg("committing results to DB")
self.params['runtime'] = aligntime
self.insertRefBasedRun(partlist, imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
### remove temporary files
apFile.removeFilePattern("alignments/alignedstack*.spi")
apFile.removeFile(finalspistack)
def runFindEM(imgdict, params, thread=False):
"""
runs a separate thread of findem.exe for each template
to get cross-correlation maps
"""
### check image
apFindEM.processAndSaveImage(imgdict, params)
dwnimgname = imgdict['filename'] + ".dwn.mrc"
if not os.path.isfile(dwnimgname):
apDisplay.printError("cound not find image to process: " + dwnimgname)
### check template
if len(params['templatelist']) < 1:
apDisplay.printError("templatelist == 0; there are no templates")
joblist = []
ccmapfilelist = []
t0 = time.time()
for i, templatename in enumerate(params['templatelist']):
classavg = i + 1
#DETERMINE OUTPUT FILE NAME
#CHANGE THIS TO BE 00%i in future
numstr = "%03d" % classavg
#numstr = str(classavg%10)+"00"
ccmapfile = "cccmaxmap" + numstr + ".mrc"
apFile.removeFile(ccmapfile)
#GET FINDEM RUN COMMANDS
feed = findEMString(classavg, templatename, dwnimgname, ccmapfile,
params)
#RUN THE PROGRAM
if thread is True:
job = findemjob(feed)
joblist.append(job)
job.start()
else:
execFindEM(feed)
#TO REMOVE LATER: Fake output
fakeOutput(dwnimgname, ccmapfile, params)
#STORE OUTPUT FILE
ccmapfilelist.append(ccmapfile)
### WAIT FOR THREADS TO COMPLETE
if thread is True:
apDisplay.printMsg("waiting for " + str(len(joblist)) +
" findem threads to complete")
for i, job in enumerate(joblist):
while job.isAlive():
sys.stderr.write(".")
time.sleep(1.5)
sys.stderr.write("\n")
apDisplay.printMsg("FindEM finished in " +
apDisplay.timeString(time.time() - t0))
for ccmapfile in ccmapfilelist:
if not os.path.isfile(ccmapfile):
apDisplay.printError("findem.exe did not run or crashed.\n")
return ccmapfilelist
def _getAllSeries(self):
startt = time.time()
self.seriestree = []
if self.params['mrcnames'] is not None:
mrcfileroot = self.params['mrcnames'].split(",")
if self.params['sessionname'] is not None:
images = apDatabase.getSpecificImagesFromSession(
mrcfileroot, self.params['sessionname'])
else:
images = apDatabase.getSpecificImagesFromDB(mrcfileroot)
elif self.params['sessionname'] is not None:
if self.params['preset'] is not None:
images = apDatabase.getImagesFromDB(self.params['sessionname'],
self.params['preset'])
else:
self.seriestree = apDatabase.getAllTiltSeriesFromSessionName(
self.params['sessionname'])
else:
if self.params['mrcnames'] is not None:
apDisplay.printMsg("MRC List: " +
str(len(self.params['mrcnames'])) + " : " +
str(self.params['mrcnames']))
apDisplay.printMsg("Session: " + str(self.params['sessionname']) +
" : " + str(self.params['preset']))
apDisplay.printError("no files specified")
# Only use finished tilt series
if len(self.seriestree) > 0:
indices_to_pop = []
for tiltseriesdata in self.seriestree:
if not apDatabase.getTiltSeriesDoneStatus(tiltseriesdata):
indices_to_pop.append(
self.seriestree.index(tiltseriesdata))
indices_to_pop.sort()
indices_to_pop.reverse()
for index in indices_to_pop:
self.seriestree.pop(index)
else:
for image in images:
tiltseriesdata = image['tilt series']
if tiltseriesdata and tiltseriesdata not in self.seriestree and apDatabase.getTiltSeriesDoneStatus(
tiltseriesdata):
self.seriestree.append(tiltseriesdata)
precount = len(self.seriestree)
apDisplay.printMsg("Found " + str(precount) + " tilt series in " +
apDisplay.timeString(time.time() - startt))
### REMOVE PROCESSED IMAGES
apDisplay.printMsg("Remove processed series")
self._removeProcessedSeries()
### SET SERIES ORDER
if self.params['reverse'] is True:
apDisplay.printMsg("Process series new to old")
else:
# by default series are new to old
apDisplay.printMsg("Process series old to new")
self.seriestree.sort(self._reverseSortSeriesTree)
### LIMIT NUMBER
if self.params['limit'] is not None:
lim = self.params['limit']
if len(self.seriestree) > lim:
apDisplay.printMsg("Limiting number of series to " + str(lim))
self.seriestree = self.seriestree[:lim]
self.stats['seriescount'] = len(self.seriestree)
def start(self):
### new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
newstack = os.path.join(self.params['rundir'], stackdata['name'])
apStack.checkForPreviousStack(newstack)
includelist = []
excludelist = []
### list of classes to be excluded
if self.params['dropclasslist'] is not None:
excludestrlist = self.params['dropclasslist'].split(",")
for excludeitem in excludestrlist:
excludelist.append(int(excludeitem.strip()))
apDisplay.printMsg("Exclude list: "+str(excludelist))
### list of classes to be included
if self.params['keepclasslist'] is not None:
includestrlist = self.params['keepclasslist'].split(",")
for includeitem in includestrlist:
includelist.append(int(includeitem.strip()))
### or read from keepfile
elif self.params['keepfile'] is not None:
keeplistfile = open(self.params['keepfile'])
for line in keeplistfile:
if self.params['excludefrom'] is True:
excludelist.append(int(line.strip()))
else:
includelist.append(int(line.strip()))
keeplistfile.close()
apDisplay.printMsg("Include list: "+str(includelist))
### get particles from align or cluster stack
apDisplay.printMsg("Querying database for particles")
q0 = time.time()
if self.params['alignid'] is not None:
# DIRECT SQL STUFF
sqlcmd = "SELECT " + \
"apd.partnum, " + \
"apd.xshift, apd.yshift, " + \
"apd.rotation, apd.mirror, " + \
"apd.spread, apd.correlation, " + \
"apd.score, apd.bad, " + \
"spd.particleNumber, " + \
"ard.refnum "+ \
"FROM ApAlignParticleData apd " + \
"LEFT JOIN ApStackParticleData spd ON " + \
"(apd.`REF|ApStackParticleData|stackpart` = spd.DEF_id) " + \
"LEFT JOIN ApAlignReferenceData ard ON" + \
"(apd.`REF|ApAlignReferenceData|ref` = ard.DEF_id) " + \
"WHERE `REF|ApAlignStackData|alignstack` = %i"%(self.params['alignid'])
# These are AlignParticles
particles = sinedon.directq.complexMysqlQuery('appiondata',sqlcmd)
elif self.params['clusterid'] is not None:
clusterpartq = appiondata.ApClusteringParticleData()
clusterpartq['clusterstack'] = self.clusterstackdata
# These are ClusteringParticles
particles = clusterpartq.query()
apDisplay.printMsg("Completed in %s\n"%(apDisplay.timeString(time.time()-q0)))
### write included particles to text file
includeParticle = []
excludeParticle = 0
badscore = 0
badshift = 0
badspread = 0
f = open("test.log", "w")
count = 0
t0 = time.time()
apDisplay.printMsg("Parsing particle information")
# find out if there is alignparticle info:
is_cluster_p = False
# alignparticle is a key of any particle in particles if the latter is
# a CluateringParticle
if 'alignparticle' in particles[0]:
is_cluster_p = True
for part in particles:
count += 1
if is_cluster_p:
# alignpart is an item of ClusteringParticle
alignpart = part['alignparticle']
try:
classnum = int(part['refnum'])-1
except:
apDisplay.printWarning("particle %d was not put into any class" % (part['partnum']))
emanstackpartnum = alignpart['stackpart']['particleNumber']-1
else:
# particle has info from AlignedParticle as results of direct query
alignpart = part
try:
classnum = int(alignpart['refnum'])-1
except:
apDisplay.printWarning("particle %d was not put into any class" % (part['partnum']))
classnum = None
emanstackpartnum = int(alignpart['particleNumber'])-1
### check shift
if self.params['maxshift'] is not None:
shift = math.hypot(alignpart['xshift'], alignpart['yshift'])
if shift > self.params['maxshift']:
excludeParticle += 1
if classnum is not None:
f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum))
else:
f.write("%d\t%d\texclude\n"%(count, emanstackpartnum))
badshift += 1
continue
if self.params['minscore'] is not None:
### check score
if ( alignpart['score'] is not None
and alignpart['score'] < self.params['minscore'] ):
excludeParticle += 1
if classnum is not None:
f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum))
else:
f.write("%d\t%d\texclude\n"%(count, emanstackpartnum))
badscore += 1
continue
### check spread
if ( alignpart['spread'] is not None
and alignpart['spread'] < self.params['minscore'] ):
excludeParticle += 1
if classnum is not None:
f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum))
else:
f.write("%d\t%d\texclude\n"%(count, emanstackpartnum))
badspread += 1
continue
if classnum is not None:
if includelist and (classnum in includelist):
includeParticle.append(emanstackpartnum)
f.write("%d\t%d\t%d\tinclude\n"%(count, emanstackpartnum, classnum))
elif excludelist and not (classnum in excludelist):
includeParticle.append(emanstackpartnum)
f.write("%d\t%d\t%d\tinclude\n"%(count, emanstackpartnum, classnum))
else:
excludeParticle += 1
f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum))
else:
excludeParticle += 1
f.write("%d\t%d\texclude\n"%(count, emanstackpartnum))
f.close()
includeParticle.sort()
if badshift > 0:
apDisplay.printMsg("%d paricles had a large shift"%(badshift))
if badscore > 0:
apDisplay.printMsg("%d paricles had a low score"%(badscore))
if badspread > 0:
apDisplay.printMsg("%d paricles had a low spread"%(badspread))
apDisplay.printMsg("Completed in %s\n"%(apDisplay.timeString(time.time()-t0)))
apDisplay.printMsg("Keeping "+str(len(includeParticle))+" and excluding "+str(excludeParticle)+" particles")
### write kept particles to file
self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list")
apDisplay.printMsg("writing to keepfile "+self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
for partnum in includeParticle:
kf.write(str(partnum)+"\n")
kf.close()
### get number of particles
numparticles = len(includeParticle)
if excludelist:
self.params['description'] += ( " ... %d particle substack with %s classes excluded"
% (numparticles, self.params['dropclasslist']))
elif includelist:
self.params['description'] += ( " ... %d particle substack with %s classes included"
% (numparticles, self.params['keepclasslist']))
outavg = os.path.join(self.params['rundir'],"average.mrc")
### create the new sub stack
# first check if virtual stack
if not os.path.isfile(oldstack):
vstackdata=apStack.getVirtualStackParticlesFromId(self.params['stackid'])
vparts = vstackdata['particles']
oldstack = vstackdata['filename']
# get subset of virtualstack
vpartlist = [int(vparts[p]['particleNumber'])-1 for p in includeParticle]
if self.params['writefile'] is True:
apStack.makeNewStack(oldstack, newstack, vpartlist, bad=self.params['savebad'])
apStack.averageStack(stack=oldstack,outfile=outavg,partlist=vpartlist)
else:
if self.params['writefile'] is True:
apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=self.params['savebad'])
apStack.averageStack(stack=oldstack,outfile=outavg,partlist=includeParticle)
if self.params['writefile'] is True and not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
if self.params['commit'] is True:
apStack.commitSubStack(self.params,included=includeParticle)
newstackid = apStack.getStackIdFromPath(newstack)
apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=4)
def start(self):
### get original aligned stack name
astack = self.analysisdata['alignstack']['imagicfile']
### spider has problems with file name if it includes an "x#"
astack = re.sub(r'x(\d)', r'x-\1', astack)
### get original align stack
imagicalignedstack = os.path.join(
self.analysisdata['alignstack']['path']['path'], astack)
alignedstack = re.sub("\.", "_", imagicalignedstack) + ".spi"
while os.path.isfile(alignedstack):
apFile.removeFile(alignedstack)
emancmd = "proc2d %s %s spiderswap" % (imagicalignedstack,
alignedstack)
apEMAN.executeEmanCmd(emancmd, showcmd=True, verbose=True)
### get database information
numpart = self.analysisdata['alignstack']['num_particles']
corandata = os.path.join(self.analysisdata['path']['path'],
"coran/corandata")
### parse factor list
factorlist = self.params['factorstr'].split(",")
factorstr, factorkey = operations.intListToString(factorlist)
factorstr = re.sub(",", ", ", factorstr)
apDisplay.printMsg("using factorlist " + factorstr)
if len(factorlist) > self.analysisdata['coranrun']['num_factors']:
apDisplay.printError(
"Requested factor list is longer than available factors")
if self.params['commit'] is True:
self.insertClusterRun(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
numclasslist = self.params['numclasslist'].split(",")
if self.params['method'] != "kmeans":
rundir = "cluster"
apParam.createDirectory(rundir)
### step 1: use coran data to create hierarchy
dendrogramfile = classification.hierarchClusterProcess(
numpart, factorlist, corandata, rundir, dataext=".spi")
### step 2: asssign particles to groups based on hierarchy
for item in numclasslist:
t0 = time.time()
if not item or not re.match("^[0-9]+$", item):
continue
numclass = int(item)
apDisplay.printColor(
"\n============================\nprocessing class averages for "
+ str(numclass) + " classes\n============================\n",
"green")
#run the classification
if self.params['method'] == "kmeans":
apDisplay.printMsg("Using the k-means clustering method")
classavg, classvar = classification.kmeansCluster(
alignedstack,
numpart,
numclasses=numclass,
timestamp=self.timestamp,
factorlist=factorlist,
corandata=corandata,
dataext=".spi")
else:
apDisplay.printMsg("Using the hierarch clustering method")
classavg, classvar = classification.hierarchClusterClassify(
alignedstack,
dendrogramfile,
numclass,
self.timestamp,
rundir,
dataext=".spi")
#classavg,classvar = classification.hierarchCluster(alignedstack, numpart, numclasses=numclass,
# timestamp=self.timestamp, factorlist=factorlist, corandata=corandata, dataext=".spi")
if self.params['commit'] is True:
self.insertClusterStack(classavg,
classvar,
numclass,
insert=True)
else:
apDisplay.printWarning("not committing results to DB")
apDisplay.printMsg("Completed " + str(numclass) + " classes in " +
apDisplay.timeString(time.time() - t0))
def normalizeVol(volfile, dataext=".spi"):
"""
inputs:
volume
outputs:
volume
"""
### setup
starttime = time.time()
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(volfile+dataext):
apDisplay.printError("volume file not found: "+volfile+dataext)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
### read out the statistics of the volume
mySpider.toSpider("FS x11,x12",
volfile, #filename for volume
)
mySpider.toSpider("IF(x12.LT.0.0)x12=-x12")
### set all values to positive
mySpider.toSpider("AR",
volfile, #filename for volume
"_1",
"(P1+x12)",
)
### save file
mySpider.toSpider("CP",
"_1",
volfile, #filename for volume
)
mySpider.close()
apDisplay.printColor("finished normalizing the volume to set all values to be positive"+apDisplay.timeString(time.time()-starttime), "cyan")
return
def removePtclsByJumps(self, particles, rejectlst):
eulerjump = apEulerJump.ApEulerJump()
numparts = len(particles)
apDisplay.printMsg("finding euler jumps for " + str(numparts) +
" particles")
### check symmetry
symmetry = eulerjump.getSymmetry(self.params['reconid'], msg=True)
if not re.match("^[cd][0-9]+$", symmetry.lower()) and not re.match(
"^icos", symmetry.lower()):
apDisplay.printError(
"Cannot calculate euler jumps for symmetry: " + symmetry)
return
self.params['sym'] = symmetry.lower()
### prepare file
f = open('jumps.txt', 'w', 0666)
f.write("#pnum\t")
headerlist = ('mean', 'median', 'stdev', 'min', 'max')
for key in headerlist:
f.write(key + "\t")
f.write("\n")
### get stack particles
stackparts = apStack.getStackParticlesFromId(self.params['stackid'])
### start loop
t0 = time.time()
medians = []
count = 0
apDisplay.printMsg("processing euler jumps for recon run=" +
str(self.params['reconid']))
for stackpart in stackparts:
count += 1
partnum = stackpart['particleNumber']
f.write('%d\t' % partnum)
jumpdata = eulerjump.getEulerJumpData(
self.params['reconid'],
stackpartid=stackpart.dbid,
stackid=self.params['stackid'],
sym=symmetry)
medians.append(jumpdata['median'])
if (jumpdata['median'] >
self.params['avgjump']) and partnum not in rejectlst:
rejectlst.append(partnum)
for key in headerlist:
f.write("%.3f\t" % (jumpdata[key]))
f.write("\n")
if count % 1000 == 0:
timeremain = (time.time() - t0) / (count + 1) * (numparts -
count)
apDisplay.printMsg(
"particle=% 5d; median jump=% 3.2f, remain time= %s" %
(partnum, jumpdata['median'],
apDisplay.timeString(timeremain)))
#f.flush()
### print stats
apDisplay.printMsg("-- median euler jumper stats --")
medians = numpy.asarray(medians, dtype=numpy.float32)
apDisplay.printMsg("mean/std :: " + str(round(medians.mean(), 2)) +
" +/- " + str(round(medians.std(), 2)))
apDisplay.printMsg("min/max :: " + str(round(medians.min(), 2)) +
" <> " + str(round(medians.max(), 2)))
perrej = round(
100.0 * float(numparts - len(rejectlst)) / float(numparts), 2)
apDisplay.printMsg("keeping " + str(numparts - len(rejectlst)) +
" of " + str(numparts) + " particles (" +
str(perrej) + "%) so far " + " in " +
apDisplay.timeString(time.time() - t0))
return rejectlst
def radonAlign(self, stackfile):
"""
performs the meat of the program aligning the particles and creating references
"""
### FUTURE: only read a few particles into memory at one time
imageinfo = apImagicFile.readImagic(stackfile, msg=False)
imagelist = imageinfo['images']
reflist = self.createReferences(imagelist)
radonimagelist = self.getRadons(imagelist)
### a pre-normalization value so the reference pixels do not overflow
partperref = self.params['numpart'] / float(self.params['numrefs'])
for iternum in range(self.params['numiter']):
### save references to a file
apImagicFile.writeImagic(reflist,
"reflist%02d.hed" % (iternum),
msg=False)
### create Radon transforms for references
radonreflist = self.getRadons(reflist)
### create empty references
newreflist = []
newrefcount = []
shape = imagelist[0].shape
for i in range(self.params['numrefs']):
newrefcount.append(0)
newreflist.append(numpy.zeros(shape))
### get alignment parameters
aligndatalist = []
cclist = []
t0 = time.time()
for i in range(len(imagelist)):
if i % 50 == 0:
### FUTURE: add time estimate
sys.stderr.write(".")
image = imagelist[i]
radonimage = radonimagelist[i]
aligndata = self.getBestAlignForImage(image, radonimage,
reflist, radonreflist,
None)
#aligndatalist.append(aligndata)
refid = aligndata['refid']
cclist.append(aligndata['bestcc'])
### create new references
refimage = reflist[refid]
alignedimage = self.transformImage(image, aligndata, refimage)
newreflist[refid] += alignedimage / partperref
newrefcount[refid] += 1
sys.stderr.write("\n")
print "Alignment complete in %s" % (
apDisplay.timeString(time.time() - t0))
### report median cross-correlation, it should get better each iter
mediancc = numpy.median(numpy.array(cclist))
apDisplay.printMsg("Iter %02d, Median CC: %.8f" %
(iternum, mediancc))
print newrefcount
### FUTURE: re-calculate Radon transform for particles with large shift
### new references are now the old references
shape = reflist[0].shape
reflist = []
for i in range(self.params['numrefs']):
if newrefcount[i] == 0:
### reference with no particles -- just add noise
apDisplay.printWarning("Reference %02d has no particles" %
(i + 1))
ref = numpy.random.random(shape)
else:
ref = (newreflist[i] / newrefcount[i]) * partperref
reflist.append(ref)
return aligndatalist
def start(self):
self.runtime = 0
self.checkCoranRun()
### convert stack to spider
self.alignstackdata = self.getAlignedStack()
maskpixrad = self.params['maskrad'] / self.alignstackdata[
'pixelsize'] / self.params['bin']
boxpixdiam = int(math.ceil(maskpixrad) + 1) * 2
if boxpixdiam * self.params['bin'] > self.alignstackdata['boxsize']:
boxpixdiam = math.floor(self.alignstackdata['boxsize'] /
self.params['bin'])
clippixdiam = boxpixdiam * self.params['bin']
apDisplay.printMsg("Pixel mask radius=" + str(maskpixrad))
oldalignedstack = os.path.join(self.alignstackdata['path']['path'],
self.alignstackdata['imagicfile'])
alignedstackname = re.sub("\.", "_",
self.alignstackdata['imagicfile']) + ".spi"
alignedstack = os.path.join(self.params['rundir'], alignedstackname)
apFile.removeFile(alignedstack)
emancmd = ("proc2d %s %s spiderswap shrink=%d clip=%d,%d edgenorm" %
(oldalignedstack, alignedstack, self.params['bin'],
clippixdiam, clippixdiam))
if self.params['numpart'] is not None:
emancmd += " last=%d" % (self.params['numpart'] - 1)
numpart = self.params['numpart']
else:
numpart = self.getNumAlignedParticles()
apEMAN.executeEmanCmd(emancmd, verbose=True)
esttime = classification.estimateTime(numpart, maskpixrad)
apDisplay.printColor("Running spider this can take awhile, estimated time: "+\
apDisplay.timeString(esttime),"cyan")
### do correspondence analysis
corantime = time.time()
self.contriblist = classification.correspondenceAnalysis(
alignedstack,
boxsize=boxpixdiam,
maskpixrad=maskpixrad,
numpart=numpart,
numfactors=self.params['numfactors'])
corantime = time.time() - corantime
### make dendrogram
dendrotime = time.time()
classification.makeDendrogram(
numfactors=min(3, self.params['numfactors']))
dendrotime = time.time() - dendrotime
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = corantime
self.insertCoranRun(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apFile.removeFile(alignedstack, warn=True)
apDisplay.printMsg("Correspondence Analysis time: " +
apDisplay.timeString(corantime))
apDisplay.printMsg("Make Dendrogram time: " +
apDisplay.timeString(dendrotime))
apDisplay.printMsg("Database Insertion time: " +
apDisplay.timeString(inserttime))
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.clipsize = int(
math.floor(self.stack['boxsize'] / self.params['bin'] / 2.0) *
self.params['bin'] * 2.0)
#self.checkRunNamePath()
### convert stack to spider
spiderstack = self.createSpiderFile()
#create template stack
templatestack = self.createTemplateStack()
#create orientation reference
orientref = self.createOrientationReference()
###################################################################
aligntime = time.time()
### create batch file
batchfilepath = self.setupBatchFile(spiderstack, templatestack,
orientref)
### run the spider alignment
apDisplay.printColor(
"Running iterative ref-classification and free-alignment with spider",
"cyan")
self.runSpiderBatch(batchfilepath, spiderstack)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
###################################################################
### remove unaligned spider stack
apDisplay.printMsg("Removing un-aligned stack: " + spiderstack)
apFile.removeFile(spiderstack, warn=False)
### check to be sure files exist
avgfile = os.path.join(self.params['rundir'],
"alignstack.spi") #class averages
if not os.path.isfile(avgfile):
apDisplay.printError(
"Final stack of aligned particles does not exist.")
### convert stacks to imagic
self.convertStack2Imagic("alignstack.spi")
self.convertStack2Imagic("avg.spi")
### make alignment average in mrc format
emancmd = "proc2d avg.spi average.mrc average"
apEMAN.executeEmanCmd(emancmd)
inserttime = time.time()
if self.params['commit'] is True:
apDisplay.printWarning("committing results to DB")
self.runtime = aligntime
self.insertEdIterRun(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: " +
apDisplay.timeString(inserttime))
def convertSQLtoTree(self, results, notstackid):
t0 = time.time()
parttree = []
minpartnum1 = 1e6
minpartnum2 = 1e6
count = 0
tiltangles = []
notangles = []
for row in results:
if len(row) < 6:
apDisplay.printError("delete MySQL cache file and run again")
try:
partnum1 = self.intOrNone(row[0])
stackid1 = self.intOrNone(row[1])
angle1 = self.floatOrNone(row[2])
partnum2 = self.intOrNone(row[3])
stackid2 = self.intOrNone(row[4])
angle2 = self.floatOrNone(row[5])
if partnum1 and partnum1 < minpartnum1:
minpartnum1 = partnum1
if partnum2 and partnum2 < minpartnum2:
minpartnum2 = partnum2
if partnum1 is not None and partnum2 is not None:
if stackid1 == notstackid:
partpair1 = {
'part1': partnum1,
'part2': -partnum2,
'stackid1': stackid1,
'stackid2': stackid2,
'tilt': angle1
}
notangles.append(angle1)
partpair2 = {
'part1': -partnum2,
'part2': partnum1,
'stackid1': stackid2,
'stackid2': stackid1,
'tilt': angle2
}
tiltangles.append(angle2)
else:
partpair1 = {
'part1': -partnum1,
'part2': partnum2,
'stackid1': stackid1,
'stackid2': stackid2,
'tilt': angle1
}
tiltangles.append(angle1)
partpair2 = {
'part1': partnum2,
'part2': -partnum1,
'stackid1': stackid2,
'stackid2': stackid1,
'tilt': angle2
}
notangles.append(angle2)
elif partnum1 is not None:
if stackid1 == notstackid:
partpair1 = {
'part1': partnum1,
'part2': None,
'stackid1': stackid1,
'stackid2': None,
'tilt': angle1
}
else:
partpair1 = {
'part1': -partnum1,
'part2': None,
'stackid1': stackid1,
'stackid2': None,
'tilt': angle1
}
elif partnum2 is not None:
if stackid2 == notstackid:
partpair2 = {
'part1': partnum2,
'part2': None,
'stackid1': stackid2,
'stackid2': None,
'tilt': angle2
}
else:
partpair2 = {
'part1': -partnum2,
'part2': None,
'stackid1': stackid2,
'stackid2': None,
'tilt': angle2
}
if partnum1 is not None:
parttree.append(partpair1)
count += 1
if partnum2 is not None:
parttree.append(partpair2)
count += 1
except:
print count, row
apDisplay.printError("bad row entry")
print count, minpartnum1, minpartnum2
apDisplay.printMsg("Converted " + str(len(parttree)) +
" particles in " +
apDisplay.timeString(time.time() - t0))
tiltarray = numpy.array(tiltangles, dtype=numpy.float32)
self.tiltangle = tiltarray.mean()
apDisplay.printColor(
" Tilted angle: %.3f +/- %.3f" %
(self.tiltangle, tiltarray.std()), "cyan")
print tiltarray
notarray = numpy.array(notangles, dtype=numpy.float32)
self.notangle = notarray.mean()
apDisplay.printColor(
"Untilted angle: %.3f +/- %.3f" % (self.notangle, notarray.std()),
"green")
print notarray
return parttree
def getParticles(self, notstackid, tiltstackid):
cachefile = "parttree-" + str(notstackid) + "_" + str(
tiltstackid) + ".cache"
if os.path.isfile(cachefile):
cachef = open(cachefile, "r")
parttree = cPickle.load(cachef)
cachef.close()
return parttree
t0 = time.time()
query = (
"SELECT \n" + " stpart1.`particleNumber` AS partnum1, \n" +
" stpart1.`REF|ApStackData|stack` AS stackid1, \n" +
" DEGREES(scoped1.`SUBD|stage position|a`) AS alpha1, \n" +
" stpart2.`particleNumber` AS partnum2, \n" +
" stpart2.`REF|ApStackData|stack` AS stackid2, \n" +
" DEGREES(scoped2.`SUBD|stage position|a`) AS alpha2 \n" +
"FROM `ApTiltParticlePairData` AS tiltd \n" +
"LEFT JOIN `ApStackParticleData` AS stpart1 \n" +
" ON stpart1.`REF|ApParticleData|particle` = tiltd.`REF|ApParticleData|particle1` \n"
+ "LEFT JOIN `ApParticleData` AS part1 \n" +
" ON stpart1.`REF|ApParticleData|particle` = part1.`DEF_id` \n" +
"LEFT JOIN dbemdata.`AcquisitionImageData` AS imaged1 \n" +
" ON part1.`REF|leginondata|AcquisitionImageData|image` = imaged1.`DEF_id` \n"
+ "LEFT JOIN dbemdata.`ScopeEMData` AS scoped1 \n" +
" ON imaged1.`REF|ScopeEMData|scope` = scoped1.`DEF_id` \n" +
"LEFT JOIN `ApStackParticleData` AS stpart2 \n" +
" ON stpart2.`REF|ApParticleData|particle` = tiltd.`REF|ApParticleData|particle2` \n"
+ "LEFT JOIN `ApParticleData` AS part2 \n" +
" ON stpart2.`REF|ApParticleData|particle` = part2.`DEF_id` \n" +
"LEFT JOIN dbemdata.`AcquisitionImageData` AS imaged2 \n" +
" ON part2.`REF|leginondata|AcquisitionImageData|image` = imaged2.`DEF_id` \n"
+ "LEFT JOIN dbemdata.`ScopeEMData` AS scoped2 \n" +
" ON imaged2.`REF|ScopeEMData|scope` = scoped2.`DEF_id` \n" +
"WHERE \n" + " ( ( stpart1.`REF|ApStackData|stack` = " +
str(notstackid) + " \n" +
" OR stpart1.`REF|ApStackData|stack` IS NULL ) \n" +
" AND ( stpart2.`REF|ApStackData|stack` = " + str(tiltstackid) +
" \n" + " OR stpart2.`REF|ApStackData|stack` IS NULL ) )\n" +
" OR ( ( stpart1.`REF|ApStackData|stack` = " + str(tiltstackid) +
" \n" + " OR stpart1.`REF|ApStackData|stack` IS NULL ) \n" +
" AND ( stpart2.`REF|ApStackData|stack` = " + str(notstackid) +
" \n" + " OR stpart2.`REF|ApStackData|stack` IS NULL ) )\n" +
"ORDER BY stpart1.`particleNumber` ASC \n"
#+"LIMIT 21 \n"
)
apDisplay.printMsg("particle query at " + time.asctime())
self.cursor.execute(query)
results = self.cursor.fetchall()
if not results:
apDisplay.printError("Failed to get stack particles")
apDisplay.printMsg("Fetched " + str(len(results)) + " data pairs in " +
apDisplay.timeString(time.time() - t0))
parttree = self.convertSQLtoTree(results, notstackid)
parttree.sort(self.compPart)
### save to file
cachef = open(cachefile, "w")
cPickle.dump(parttree, cachef)
cachef.close()
#for part in parttree:
# if part['part2'] is None:
# print part
#print ""
#print ""
return parttree
def preprocessStack(self):
'''
Use database particle stack file to create a stack file with binning and filtering
ready for processing.
'''
need_modify = False
emancmd = "proc2d "
if not os.path.isfile(self.stack['file']):
apDisplay.printError("stackfile does not exist: " +
self.stack['file'])
emancmd += self.stack['file'] + " "
extname, addformat = self.proc2dFormatConversion()
outstack = os.path.join(self.params['rundir'], "start.%s" % extname)
apFile.removeFile(outstack, warn=True)
emancmd += outstack + " "
stackfilenamebits = self.stack['file'].split('.')
oldextname = stackfilenamebits[-1]
if extname != oldextname:
need_modify = True
if addformat:
need_modify = True
emancmd += addformat + " "
emancmd += "apix=" + str(self.stack['apix']) + " "
if self.params['lowpass'] > 0:
need_modify = True
emancmd += "lp=" + str(self.params['lowpass']) + " "
if self.params['highpass'] > 0:
need_modify = True
emancmd += "hp=" + str(self.params['highpass']) + " "
if self.params['last'] > 0 and self.params['last'] < self.params[
'totalpart']:
need_modify = True
emancmd += "last=" + str(self.params['last'] - 1) + " "
if self.params['bin'] > 1:
need_modify = True
emancmd += "shrink=" + str(self.params['bin']) + " "
clipsize = self.calcClipSize(self.stack['boxsize'],
self.params['bin'])
emancmd += "clip=" + str(clipsize) + "," + str(
clipsize) + " edgenorm" + " "
self.stack['boxsize'] = clipsize / self.params['bin']
if self.invert:
need_modify = True
emancmd += 'invert '
if need_modify:
apFile.removeStack(outstack, warn=False)
starttime = time.time()
apDisplay.printColor(
"Running particle stack conversion.... This can take a while",
"cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor(
"finished eman in " +
apDisplay.timeString(time.time() - starttime), "cyan")
else:
# no need to execute EmanCmd if the stack is not modified
apDisplay.printColor(
"No stack pre-processing is needed. Copying stack to run directory.",
"cyan")
shutil.copy(self.stack['file'], outstack)
outstackimg = outstack.replace('hed', 'img')
if not os.path.isfile(outstackimg):
# only copy if not exist to save time
shutil.copy(self.stack['file'].replace('hed', 'img'),
outstackimg)
self.stack['file'] = outstack
if not os.path.isfile(self.stack['file']):
apDisplay.printColor(
"Could not locate stack: %s" % self.stack['file'], "cyan")
self.stack['apix'] = self.stack['apix'] * self.params['bin']
return outstack
def start(self):
### simple is written in Fortran, which cannot take inputs of certain length, therefore one needs
### to change to the directory to minimize the filename length, in particular for the stack
os.chdir(self.params['rundir'])
### stack needs to be centered
if self.params['no_center'] is False:
if os.path.isfile(os.path.join(self.params['rundir'], "ali.hed")):
apFile.removeStack(
os.path.join(self.params['rundir'], "ali.hed"))
centstack = os.path.join(self.params['rundir'], "ali.hed")
centcmd = "cenalignint %s > cenalignint.log" % (self.stack['file'])
apParam.runCmd(centcmd, "EMAN")
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".spi")
if os.path.isfile(self.params['localstack']):
apFile.removeFile(self.params['localstack'])
if self.params['no_center'] is False:
proccmd = "proc2d " + centstack + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
else:
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
self.boxsize, self.boxsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
proccmd += " spiderswap"
# if self.params['highpass'] is not None and self.params['highpass'] > 1:
# proccmd += " hp="+str(self.params['highpass'])
# if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
# proccmd += " lp="+str(self.params['lowpass'])
apParam.runCmd(proccmd, "EMAN", verbose=True)
# if self.params['numpart'] != int(spider.getSpiderHeader(self.params['localstack'])[-2]):
# apDisplay.printError("Missing particles in stack")
### setup Simple command
aligntime = time.time()
simpleopts = (
"" + " stk=%s" % os.path.basename(self.params['localstack']) +
" box=%d" % self.boxsize + " nptcls=%d" % self.params['numpart'] +
" smpd=%.3f" % self.apix + " ring2=%d" % self.params['ring2'] +
" ncls=%d" % self.params['ncls'] +
" minp=%d" % self.params['minp'] +
" nvars=%d" % self.params['nvars'] +
" nthr=%d" % self.params['nproc'])
if self.params['no_kmeans'] is True:
simpleopts += " kmeans=off"
if self.params['nran'] is not None:
simpleopts += "nran=%d" % self.params['nran']
### SIMPLE 2D clustering
apDisplay.printColor(
"Using " + str(self.params['nproc']) + " processors!", "green")
simpleexe = apParam.getExecPath("cluster", die=True)
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd,
package="SIMPLE",
verbose=True,
showcmd=True,
logfile="cluster.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment & Classification time: " +
apDisplay.timeString(self.params['runtime']))
### SIMPLE spider to Fourier format
clsavgs = "cavgstk.spi"
if not os.path.isfile(os.path.join(self.params['rundir'], clsavgs)):
apDisplay.printError(
"class averages were not created! try rerunning with centering, more particles, or less ppc"
)
try:
nptcls = spider.getSpiderHeader(clsavgs)[-2]
except:
nptcls = self.params['ncls']
apDisplay.printWarning(
"class average file may not have been created! Please check existence of file cavgstk.spi"
)
projfile = "projs"
projext = ".fim"
simpleexe = apParam.getExecPath("spi_to_fim", die=True)
simpleopts = ("" + " stk=%s" % clsavgs + " box=%d" % self.boxsize +
" nptcls=%d" % nptcls + " smpd=%.3f" % self.apix +
" outbdy=%s" % projfile +
" msk=%d" % self.params['mask'])
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd,
package="SIMPLE",
verbose=True,
showcmd=True,
logfile="spi_to_fim.std")
### SIMPLE origami, ab initio 3D reconstruction
refinetime = time.time()
simpleexe = apParam.getExecPath("origami", die=True)
simpleopts = (
"" + " fstk=%s" % projfile + projext +
" froms=%d" % self.params['froms'] +
" tos=%d" % self.params['tos'] + " lp=%d" % self.params['lp'] +
" hp=%d" % self.params['hp'] +
" maxits=%d" % self.params['maxits'] +
" msk=%d" % self.params['mask'] + " mw=%d" % self.params['mw'] +
" frac=%.3f" % self.params['frac'] +
" amsklp=%d" % self.params['amsklp'] +
" edge=%d" % self.params['edge'] + " trs=%d" % self.params['trs'] +
" nthr=%d" % self.params['nproc'])
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd,
package="SIMPLE",
verbose=True,
showcmd=True,
logfile="origami.std")
refinetime = time.time() - refinetime
apDisplay.printMsg("Origami reconstruction time: " +
apDisplay.timeString(refinetime))
# '''
### minor post-processing
self.clearIntermediateFiles()
apParam.dumpParameters(
self.params,
"simple-" + self.params['timestamp'] + "-params.pickle")
### upload results
self.runparams = apParam.readRunParameters("simple-" +
self.params['timestamp'] +
"-params.pickle")
### create average of aligned and clustered stacks, convert to IMAGIC
alignedStackSpi = "inplalgnstk.spi"
alignedStack = "inplalgnstk.hed"
if os.path.isfile(alignedStack):
apFile.removeStack(alignedStack)
emancmd = "proc2d %s %s flip" % (alignedStackSpi, alignedStack)
apParam.runCmd(emancmd, "EMAN")
clusterStackSpi = "cavgstk.spi"
clusterStack = "cavgstk.hed"
if os.path.isfile(clusterStack):
apFile.removeStack(clusterStack)
emancmd = "proc2d %s %s flip" % (clusterStackSpi, clusterStack)
apParam.runCmd(emancmd, "EMAN")
# apStack.averageStack(alignedStack)
### parse alignment and classification results
if self.params['no_center'] is False:
self.alignD = self.getAlignParameters(centparams="cenalignint.log")
else:
self.alignD = self.getAlignParameters()
if self.params['no_kmeans'] is False:
self.classD = self.getClassification("kmeans.spi", clusterStack)
else:
self.classD = self.getClassification("hcl.spi", clusterStack)
### upload to database
self.insertSIMPLEAlignParamsIntoDatabase()
self.insertAlignStackRunIntoDatabase(alignedStack, clusterStack)
self.calcResolution(alignedStack)
self.insertAlignParticlesIntoDatabase()
self.insertClusterRunIntoDatabase()
self.insertClusterStackIntoDatabase(clusterStack, len(self.classD))
self.insertSIMPLEOrigamiParamsIntoDatabase()
def power(image, pixelsize, fieldsize=None, mask_radius=0.5, msg=True):
"""
computes power spectra of image using sub-field averaging
image - (2d numpy float array) image to compute power spectra
pixelsize - (float) used to compute frequency, freq.
can be either Angstroms or meters, but freq will have same inverse units
fieldsize - (integer) size of box
mask_radius - (float) passed to imagefun.power(),
creates a mask of size mask_radius in the center
TODO: add median flag, requires saving individual power spectra rather than summing
"""
if fieldsize is None:
fieldsize = getFieldSize(image.shape)
t0 = time.time()
xsize, ysize = image.shape
xnumstep = int(math.floor(xsize/float(fieldsize)))*2-1
ynumstep = int(math.floor(ysize/float(fieldsize)))*2-1
f = fieldsize
#powersum = numpy.zeros((fieldsize,fieldsize))
#envelop = numpy.ones((fieldsize,fieldsize))
envelop = twodHann(fieldsize)
count = 0
psdlist = []
if msg is True:
sys.stderr.write("Computing power spectra in %dx%d blocks"%(fieldsize,fieldsize))
for i in range(xnumstep):
for j in range(ynumstep):
count += 1
x1 = f*i/2
x2 = x1 + f
y1 = f*j/2
y2 = y1 + f
if debug is True:
print "%03d: %d:%d, %d:%d"%(count, x1, x2, y1, y2)
elif msg is True:
sys.stderr.write(".")
cutout = image[x1:x2, y1:y2]
powerspec = imagefun.power(cutout*envelop, mask_radius)
psdlist.append(powerspec)
if xsize%fieldsize > fieldsize*0.1:
for j in range(ynumstep):
count += 1
x1 = xsize-f
x2 = xsize
y1 = f*j/2
y2 = y1 + f
if debug is True:
print "%03d: %d:%d, %d:%d"%(count, x1, x2, y1, y2)
elif msg is True:
sys.stderr.write(".")
cutout = image[x1:x2, y1:y2]
powerspec = imagefun.power(cutout*envelop, mask_radius)
psdlist.append(powerspec)
if ysize%fieldsize > fieldsize*0.1:
for i in range(xnumstep):
count += 1
x1 = f*i/2
x2 = x1 + f
y1 = ysize-f
y2 = ysize
if debug is True:
print "%03d: %d:%d, %d:%d"%(count, x1, x2, y1, y2)
elif msg is True:
sys.stderr.write(".")
cutout = image[x1:x2, y1:y2]
powerspec = imagefun.power(cutout*envelop, mask_radius)
psdlist.append(powerspec)
sys.stderr.write("\n")
freq = 1.0/(powerspec.shape[0]*pixelsize)
#poweravg = numpy.array(psdlist).mean(0)
apDisplay.printMsg("Computing median of power spectra series")
poweravg = numpy.median(psdlist, axis=0)
if msg is True:
apDisplay.printMsg("Compute PSD with fieldsize %d and %d images complete in %s"
%(fieldsize, count, apDisplay.timeString(time.time()-t0)))
return poweravg, freq
def mergeStacks(stacklist, mergestack, msg=True):
### initialization
t0 = time.time()
apFile.removeStack(mergestack, warn=msg)
root = os.path.splitext(mergestack)[0]
mergeheader = root + ".hed"
mergedata = root + ".img"
### merge data files
fout = file(mergedata, 'wb')
numpart = 0
totalsize = 0
for stackfile in stacklist:
stackdatafile = os.path.splitext(stackfile)[0] + ".img"
### size checks
npart = apFile.numImagesInStack(stackdatafile)
size = apFile.fileSize(stackdatafile)
if msg is True:
apDisplay.printMsg("%d particles in %s (%s)" %
(npart, stackdatafile, apDisplay.bytes(size)))
totalsize += size
numpart += npart
fin = file(stackdatafile, 'rb')
shutil.copyfileobj(fin, fout, 65536)
fin.close()
fout.close()
if numpart < 1:
apDisplay.printError("found %d particles" % (numpart))
if msg is True:
apDisplay.printMsg("found %d particles" % (numpart))
finalsize = apFile.fileSize(mergedata)
if finalsize != totalsize:
apDisplay.printError(
"size mismatch %s vs. %s" %
(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
if msg is True:
apDisplay.printMsg(
"size match %s vs. %s" %
(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
### merge header files
#apDisplay.printError("not finished")
mergehead = open(mergeheader, 'wb')
partnum = 1
totalsize = 0
for stackfile in stacklist:
headerfilename = os.path.splitext(stackfile)[0] + ".hed"
headfile = open(headerfilename, 'rb')
### size checks
size = apFile.fileSize(headerfilename)
if msg is True:
apDisplay.printMsg("%s (%d kB)" % (headerfilename, size / 1024))
totalsize += size
#apDisplay.printMsg("%d\t%s"%(npart, stackfile))
i = 0
npart = apFile.numImagesInStack(stackfile)
while i < npart:
#print i, npart, partnum
### read old header
data = headfile.read(1024)
### start new string
headerstr = ""
### first image number
headerstr += intToFourByte(partnum)
### number of images, less one
headerstr += intToFourByte(numpart - partnum)
### always 0,1 ???
headerstr += intToFourByte(0)
headerstr += intToFourByte(1)
### creation date: day, month, year, hour, min, sec
headerstr += intToFourByte(time.localtime()[2])
headerstr += intToFourByte(
time.localtime()[1]) #eman always uses month-1?
headerstr += intToFourByte(time.localtime()[0])
headerstr += intToFourByte(time.localtime()[3])
headerstr += intToFourByte(time.localtime()[4])
headerstr += intToFourByte(time.localtime()[5])
### append other header info, 4 character per item
headerstr += data[10 * 4:60 * 4]
### number of z slices
headerstr += intToFourByte(1)
### first image number, EMAN does this
headerstr += intToFourByte(partnum)
### append other header info, 4 character per item
headerstr += data[62 * 4:68 * 4]
headerstr += intToFourByte(33686018)
headerstr += data[69 * 4:]
mergehead.write(headerstr)
partnum += 1
i += 1
mergehead.close()
if msg is True:
apDisplay.printMsg("wrote %d particles to file %s" %
(numpart, mergestack))
finalsize = apFile.fileSize(mergeheader)
if finalsize != totalsize:
apDisplay.printError(
"size mismatch %s vs. %s" %
(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
if msg is True:
apDisplay.printMsg(
"size match %s vs. %s" %
(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
apDisplay.printMsg(
"finished stack merge of %s in %s" %
(mergestack, apDisplay.timeString(time.time() - t0)))
%(fieldsize, count, apDisplay.timeString(time.time()-t0)))
return poweravg, freq
#===================
#===================
#===================
if __name__ == "__main__":
a = mrc.read("/home/vosslab/test.mrc")
a = imagefilter.planeRegression(a)
fullpower = imagefun.power(a)
#imagestat.printImageInfo(a)
t0 = time.time()
x = numpy.arange(6, 13)
N = 2**x
print N
for n in N:
print "====================================="
b = power(a, n)
b = imagefilter.frame_cut(b, numpy.array(b.shape)/2)
imagefile.arrayToPng(b, "%04d-field.png"%(n))
imagestat.printImageInfo(b)
bin = int(round(2**12/n))
b = imagefun.bin2(fullpower, bin)
b = imagefilter.frame_cut(b, numpy.array(b.shape)/2)
imagefile.arrayToPng(b, "%04d-binned.png"%(n))
imagestat.printImageInfo(b)
print "complete in %s"%(apDisplay.timeString(time.time()-t0))
#imagestat.printImageInfo(b)
def insertIteration(self, iteration):
refineparamsq=appiondata.ApEmanRefineIterData()
refineparamsq['ang']=iteration['ang']
refineparamsq['lpfilter']=iteration['lpfilter']
refineparamsq['hpfilter']=iteration['hpfilter']
refineparamsq['pad']=iteration['pad']
refineparamsq['EMAN_maxshift']=iteration['maxshift']
refineparamsq['EMAN_hard']=iteration['hard']
refineparamsq['EMAN_classkeep']=iteration['classkeep']
refineparamsq['EMAN_classiter']=iteration['classiter']
refineparamsq['EMAN_filt3d']=iteration['filt3d']
refineparamsq['EMAN_shrink']=iteration['shrink']
refineparamsq['EMAN_euler2']=iteration['euler2']
refineparamsq['EMAN_xfiles']=iteration['xfiles']
refineparamsq['EMAN_median']=iteration['median']
refineparamsq['EMAN_phasecls']=iteration['phasecls']
refineparamsq['EMAN_fscls']=iteration['fscls']
refineparamsq['EMAN_refine']=iteration['refine']
refineparamsq['EMAN_goodbad']=iteration['goodbad']
refineparamsq['EMAN_perturb']=iteration['perturb']
refineparamsq['MsgP_cckeep']=iteration['msgpasskeep']
refineparamsq['MsgP_minptls']=iteration['msgpassminp']
#create Chimera snapshots
fscfile = os.path.join(self.params['rundir'], "fsc.eotest."+iteration['num'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix'])
if self.params['snapfilter']:
halfres = self.params['snapfilter']
volumeDensity = 'threed.'+iteration['num']+'a.mrc'
volDensPath = os.path.join(self.params['rundir'], volumeDensity)
apChimera.filterAndChimera(volDensPath, halfres, self.params['apix'],
self.params['boxsize'], 'snapshot', self.params['contour'], self.params['zoom'],
sym=iteration['sym']['eman_name'], mass=self.params['mass'])
## uncommment this for chimera image only runs...
if self.params['chimeraonly'] is True:
return
# insert resolution data
if halfres != True:
resData = self.getResolutionData(iteration)
else:
apDisplay.printWarning("resolution reported as nan, not committing results to database")
return
if self.params['package']== 'EMAN':
refineclassavg='classes_eman.'+iteration['num']+'.img'
postrefineclassavg=None
elif self.params['package']== 'EMAN/SpiCoran':
refineclassavg='classes_eman.'+iteration['num']+'.img'
postrefineclassavg='classes_coran.'+iteration['num']+'.img'
elif self.params['package']== 'EMAN/MsgP':
refineclassavg='classes_eman.'+iteration['num']+'.img'
postrefineclassavg='classes_msgp.'+iteration['num']+'.img'
else:
apDisplay.printError("Refinement Package Not Valid")
# insert refinement results
refineq = appiondata.ApRefineIterData()
refineq['refineRun'] = self.params['refineRun']
refineq['emanParams'] = refineparamsq
refineq['iteration'] = iteration['num']
refineq['resolution'] = resData
refineq['rMeasure'] = self.getRMeasureData(iteration)
refineq['mask'] = iteration['mask']
refineq['imask'] = iteration['imask']
refineq['symmetry']=iteration['sym']
refineq['exemplar'] = False
classvar = 'classes.'+iteration['num']+'.var.img'
refineq['refineClassAverages'] = refineclassavg
refineq['postRefineClassAverages'] = postrefineclassavg
if classvar in self.params['classvars']:
refineq['classVariance'] = classvar
if volumeDensity in self.params['volumes']:
refineq['volumeDensity'] = volumeDensity
apDisplay.printMsg("inserting Refinement Data into database")
if self.params['commit'] is True:
refineq.insert()
else:
apDisplay.printWarning("not committing results to database")
#insert FSC data
fscfile = os.path.join(self.params['rundir'], "fsc.eotest."+iteration['num'])
self.insertFSC(fscfile, refineq, self.params['commit'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix'])
apDisplay.printColor("FSC 0.5 Resolution: "+str(halfres), "cyan")
# get projections eulers for iteration:
eulers = self.getEulersFromProj(iteration['num'])
# get list of bad particles for this iteration
badprtls = self.readParticleLog(self.params['rundir'], iteration['num'])
# expand cls.*.tar into temp file
clsf=os.path.join(self.params['rundir'], "cls."+iteration['num']+".tar")
#print "reading",clsf
clstar=tarfile.open(clsf)
clslist=clstar.getmembers()
clsnames=clstar.getnames()
#print "extracting",clsf,"into temp directory"
for clsfile in clslist:
clstar.extract(clsfile,self.params['tmpdir'])
clstar.close()
# for each class, insert particle alignment info into database
apDisplay.printColor("Inserting Particle Classification Data for "
+str(len(clsnames))+" classes", "magenta")
t0 = time.time()
for cls in clsnames:
self.insertRefineParticleData(cls, iteration, eulers, badprtls, refineq, len(clsnames))
apDisplay.printColor("\nFinished in "+apDisplay.timeString(time.time()-t0), "magenta")
# remove temp directory
for file in os.listdir(self.params['tmpdir']):
os.remove(os.path.join(self.params['tmpdir'],file))
os.rmdir(self.params['tmpdir'])
#create euler freq map
apDisplay.printMsg("creating euler frequency map")
refrunid = int(self.params['refineRun'].dbid)
iternum = int(iteration['num'])
if self.params['package'] != 'EMAN':
postrefine = True
else:
postrefine = False
apEulerDraw.createEulerImages(refrunid, iternum, path=self.params['rundir'], postrefine=postrefine)
return
def readImagic(filename, first=1, last=None, msg=True):
"""
Rudimentary Imagic stack reader
Could be improved with more sophisticated error testing and header parsing
Currently only reads image data as floats
Currently reads header information for only first image in stack
"""
t0 = time.time()
if first < 1:
apDisplay.printError("particle numbering starts at 1")
if last is not None and first > last:
apDisplay.printError(
"requested first particle %d is greater than last particle %d" %
(first, last))
if msg is True:
apDisplay.printMsg("reading stack from disk into memory: " +
os.path.basename(filename))
if last is not None:
apDisplay.printMsg("particles %d through %d" % (first, last))
root = os.path.splitext(filename)[0]
headerfilename = root + ".hed"
datafilename = root + ".img"
### check file size, no more than 2 GB is possible
### it takes double memory on machine to read stack
filesize = apFile.fileSize(datafilename)
if first is None and last is None and filesize > bytelimit:
apDisplay.printError("Stack is too large to read %s" %
(apDisplay.bytes(filesize)))
### read stack header
headerdict = readImagicHeader(headerfilename)
### determine amount of memory needed
partbytes = 4 * headerdict['rows'] * headerdict['lines']
if last is None:
last = headerdict['nimg']
elif last > headerdict['nimg']:
apDisplay.printWarning(
"requested particle %d from stack of length %d" %
(last, headerdict['nimg']))
last = headerdict['nimg']
numpart = last - first + 1
if partbytes * numpart > filesize:
apDisplay.printError("requested particle %d from stack of length %d" %
(last, filesize / partbytes))
if partbytes * numpart > bytelimit:
apDisplay.printError(
"Stack is too large to read %d particles, requesting %s" %
(numpart, apDisplay.bytes(partbytes * numpart)))
### read stack images
images = readImagicData(datafilename, headerdict, first, numpart)
stack = {'header': headerdict, 'images': images}
if msg is True:
apDisplay.printMsg("read %d particles equaling %s in size" %
(numpart, apDisplay.bytes(partbytes * numpart)))
apDisplay.printMsg("finished in " +
apDisplay.timeString(time.time() - t0))
return stack
def writeImagic(array, filename, msg=True):
"""
Rudimentary Imagic stack writer: requires 2D images to be in list format
Could be improved with more sophisticated error testing and header parsing
Currently only reads image data as floats
Currently reads header information for only first image in stack
Inputs:
3d numpy array (numimg x row x col) OR python list of 2d numpy arrays (row x col)
filename
Modifies:
overwrites files on disk
Outputs:
none
"""
if isinstance(array, list):
### python list of 2d numpy arrays (row x col)
if len(array) == 0:
apDisplay.printWarning("writeImagic: no particles to write")
return
try:
array = numpy.asarray(array, dtype=numpy.float32)
array = numpy.fliplr(array)
except:
boxsizes = []
for part in array:
shape = part.shape
if not shape in boxsizes:
boxsizes.append(shape)
if len(boxsizes) > 1:
apDisplay.printError(
"your particles have different boxsizes: " + str(boxsizes))
apDisplay.printError(
"unknown error in particle list to numpy array conversion")
t0 = time.time()
if msg is True:
apDisplay.printMsg("writing stack to disk from memory: " + filename)
root = os.path.splitext(filename)[0]
headerfilename = root + ".hed"
datafilename = root + ".img"
if os.path.isfile(headerfilename) or os.path.isfile(datafilename):
apDisplay.printWarning("stack files '" + headerfilename +
"' already exist")
### write header file info, and dump images to image file
i = 0
headfile = open(headerfilename, 'wb')
datafile = open(datafilename, 'wb')
partnum = 0
while i < array.shape[0]:
partimg = array[i]
avg1, stdev1, min1, max1 = getImageInfo(partimg)
partnum = i + 1
headerstr = makeHeaderStr(partnum, array.shape, avg1, stdev1, min1,
max1)
headfile.write(headerstr)
# write to imagic file
datafile.write(partimg.tostring())
i += 1
headfile.close()
datafile.close()
if partnum < 1:
apDisplay.printWarning("did not write any particles to file")
if msg is True:
apDisplay.printMsg("wrote " + str(partnum) +
" particles to header file")
apDisplay.printMsg("finished in " +
apDisplay.timeString(time.time() - t0))
return
def peakExtender(raddata, rotdata, extrema, extrematype="below"):
"""
raddata - x data in inverse Angstroms
rotdata - powerspectra data, almost normalized to 0 and 1
extrema - numpy array of peak or valley locations in inverse Angstroms
extrematype - type of extrema, must be either below or above
this program looks at the CTF data using the "known" location of the extrema
extracts their extreme values
and does a linear interpolation between the extreme points
"""
t0 = time.time()
apDisplay.printMsg("starting peak extension")
extremeindices = numpy.searchsorted(raddata, extrema)
raddatasq = raddata**2
xdata = []
ydata = []
minx = extremeindices[0]
for i in range(extremeindices.shape[0]-1):
if extremeindices[i] > raddata.shape[0]-2:
break
if extremeindices[i+1] > raddata.shape[0]-1:
extremeindices[i+1] = raddata.shape[0]-1
eindex = extremeindices[i]
if i == 0:
preveindex = int(eindex/2)
else:
preveindex = extremeindices[i-1]
nexteindex = extremeindices[i+1]
eindex1 = int(round(eindex - abs(preveindex-eindex)/2.0))
eindex2 = int(round(eindex + abs(nexteindex-eindex)/2.0))
values = rotdata[eindex1:eindex2]
if extrematype is "below":
value = values.min()
elif extrematype is "above":
value = values.max()
maxx = eindex
xdata.append(raddatasq[eindex])
ydata.append(value)
if len(xdata) < 2:
#not enough indices
if extrematype is "below":
return numpy.zeros(raddata.shape)
elif extrematype is "above":
return numpy.ones(raddata.shape)
func = scipy.interpolate.interp1d(xdata, ydata, kind='linear')
extremedata = func(raddatasq[minx:maxx])
if extrematype is "below":
if minx < 3:
startvalue = 0.0
else:
startvalue = rotdata[int(minx*0.5):minx].min()
endvalue = rotdata[maxx:].min()
elif extrematype is "above":
if minx < 3:
startvalue = 1.0
else:
startvalue = rotdata[int(minx*0.5):minx].max()
endvalue = rotdata[maxx:].max()
#print startvalue, endvalue
startdata = numpy.ones((minx)) * startvalue
enddata = numpy.ones((raddata.shape[0]-maxx)) * endvalue
extremedata = numpy.hstack( (startdata, extremedata, enddata) )
apDisplay.printColor("Peak Extension Complete in %s"
%(apDisplay.timeString(time.time()-t0)), "cyan")
return extremedata
def start(self, stackfile, partlist=None):
self.stackfile = stackfile
self.starttime = time.time()
if partlist is not None:
partlist.sort()
numrequest = len(partlist)
else:
numrequest = None
self.initValues(stackfile, numrequest)
### custom pre-loop command
self.preLoop()
first = 1
last = self.stepsize
self.index = 0
t0 = time.time()
while self.index < self.numpart and first <= self.numpart:
### print message
if self.index > 10:
esttime = (time.time() - t0) / float(self.index +
1) * float(self.numpart -
self.index)
self.message(
"partnum %d to %d of %d, %s remain" %
(first, last, self.numpart, apDisplay.timeString(esttime)))
else:
self.message("partnum %d to %d of %d" %
(first, last, self.numpart))
### read images
if partlist is None:
stackdata = readImagic(stackfile,
first=first,
last=last,
msg=False)
stackarray = stackdata['images']
else:
sublist = partlist[first - 1:last]
self.message("actual partnum %d to %d" %
(sublist[0], sublist[len(sublist) - 1]))
stackarray = readParticleListFromStack(stackfile,
sublist,
msg=False)
### process images
self.processStack(stackarray)
### check for proper implementation
if self.index == 0:
apDisplay.printError(
"No particles were processed in stack loop")
### setup for next iteration
first = last + 1
last += self.stepsize
if last > self.numpart:
last = self.numpart
### END LOOP
### check for off-one reading errors
if self.index < self.numpart - 1:
print "INDEX %d -- NUMPART %d" % (self.index, self.numpart)
apDisplay.printError("Did not properly process all particles")
### custom post-loop command
self.postLoop()
self.message("finished processing stack in " +
apDisplay.timeString(time.time() - self.starttime))
return
def modelCTFNoise(self, xdata, ctfdata, contraint="below"):
"""
Master control function to fit the CTF noise function
xdata - should be in inverse Angstroms
"""
t0 = time.time()
### need to reduce precision of the xdata
### otherwise it takes too long, with no better of a fit
xdata = xdata.astype(numpy.float32)
if self.debug is True:
apDisplay.printColor("CTF limits %.1f A -->> %.1fA"
%(1./xdata.min(), 1./xdata.max()), "cyan")
if contraint == "above":
if self.debug is True:
print "constrained above function"
contraintFunction = self.modelConstFunAbove
#filterctfdata = scipy.ndimage.maximum_filter(ctfdata, size=2)
#for i in range(1):
# filterctfdata = (filterctfdata + scipy.ndimage.maximum_filter(filterctfdata, size=2))/2.0
#firstmax = filterctfdata[0:250].max()
#filterctfdata = numpy.where(filterctfdata>firstmax, firstmax, filterctfdata)
#filterctfdata = self.upwardLeftMonotonicFilter(ctfdata)
filterctfdata = ctfdata
else:
if self.debug is True:
print "constrained below function"
contraintFunction = self.modelConstFunBelow
#filterctfdata = scipy.ndimage.minimum_filter(ctfdata, size=2)
#for i in range(1):
# filterctfdata = (filterctfdata + scipy.ndimage.minimum_filter(filterctfdata, size=2))/2.0
#firstmin = filterctfdata[0:250].min()
#filterctfdata = numpy.where(filterctfdata>firstmin, firstmin, filterctfdata)
#filterctfdata = self.downwardRightMonotonicFilter(ctfdata)
filterctfdata = ctfdata
### run the initial minimizations
namelist, valuelist, fitparamslist = self.getAllInitialParameters(xdata,
filterctfdata, contraintFunction)
### figure out which initial fit was best
if self.debug is True:
namestr = "|"
valstr = "|"
conststr = "|"
for i in range(len(valuelist)):
constrainval = contraintFunction(fitparamslist[i], xdata, filterctfdata)
namestr += apDisplay.rightPadString("%s"%(namelist[i][:15]), 15)+"|"
valstr += apDisplay.leftPadString("%.4f"%(valuelist[i]), 15)+"|"
conststr += apDisplay.leftPadString("%.4e"%(constrainval), 15)+"|"
print namestr
print valstr
print conststr
### lowest is best
minvalindex = numpy.argmin(valuelist)
constrainval = contraintFunction(fitparamslist[minvalindex], xdata, filterctfdata)
valuelist = numpy.array(valuelist)
if contraint == "below":
minconval = -1e-2
elif contraint == "above":
minconval = -1e-4
else:
minconval = -1e-3
while constrainval < minconval and valuelist.min() < 1e6:
if constrainval < 0.1 and self.debug is True:
apDisplay.printMsg("Constraint violation: %.3e < %.3e"%(constrainval, minconval))
valuelist[minvalindex] *= 1e10
minvalindex = numpy.argmin(valuelist)
constrainval = contraintFunction(fitparamslist[minvalindex], xdata, filterctfdata)
if self.debug is True:
apDisplay.printColor( namelist[minvalindex]+" is best" , "cyan")
midfitparams = fitparamslist[minvalindex]
if self.debug is True:
print ( "middle parameters (%.5e, %.5e, %.5e, %.5e, %.5e)"
%(midfitparams[0], midfitparams[1], midfitparams[2], midfitparams[3], midfitparams[4]))
midvalue = self.modelFitFun(midfitparams, xdata, ctfdata)
if self.debug is True:
print "middle function value %.10f"%(midvalue)
constrainval = contraintFunction(midfitparams, xdata, ctfdata)
print "constrained value %.10e"%(constrainval)
### run the full minimization
rhobeg = (numpy.where(numpy.abs(midfitparams)<1e-20, 1e20, numpy.abs(midfitparams))).min()/1e7
if self.debug: print "RHO begin", rhobeg
fitparams = scipy.optimize.fmin_cobyla( self.modelFitFun, midfitparams,
args=(xdata, ctfdata), cons=[contraintFunction,],
consargs=(xdata, ctfdata), rhobeg=rhobeg, rhoend=rhobeg/1e4, iprint=0, maxfun=1e8)
if self.debug is True:
print ( "final parameters (%.4e, %.4e, %.4e, %.4e, %.4e)"
%(fitparams[0], fitparams[1], fitparams[2], fitparams[3], fitparams[4]))
finalvalue = self.modelFitFun(fitparams, xdata, ctfdata)
if self.debug is True:
print "final function value %.10f"%(finalvalue)
#writeDatFile("finalvalue.dat", fitparams, xdata, ctfdata)
if finalvalue <= midvalue:
if self.debug is True:
apDisplay.printColor("Final value is better", "green")
bestfitparams = fitparams
else:
if self.debug is True:
apDisplay.printColor("Final value is worse", "red")
bestfitparams = midfitparams
z = numpy.polyfit(xdata, filterctfdata, 3)
polyfitparams = [z[3], 0.0, z[2], z[1], z[0]]
if self.debug is True:
xdatasq = xdata**2
xdatasq = numpy.arange(0, len(xdata), 1)
from matplotlib import pyplot
pyplot.plot(xdatasq, ctfdata, 'r-', )
pyplot.plot(xdatasq, filterctfdata, 'b-', )
midfitdata = self.noiseModel(midfitparams, xdata)
pyplot.plot(xdatasq, midfitdata, 'm:', )
polyfitdata = self.noiseModel(polyfitparams, xdata)
pyplot.plot(xdatasq, polyfitdata, 'y-', )
finalfitdata = self.noiseModel(fitparams, xdata)
pyplot.plot(xdatasq, finalfitdata, 'k-', )
pyplot.show()
pyplot.clf()
"""
datadiff1 = scipy.ndimage.median_filter(numpy.diff(ctfdata), 3)
datadiff2 = scipy.ndimage.median_filter(numpy.diff(datadiff1), 27)
pyplot.plot(xdatasq[:500], (datadiff2/datadiff2.std())[:500], 'y-', )
pyplot.plot(xdatasq[:500], (ctfdata - ctfdata.mean())[:500], 'r-', )
pyplot.plot(xdatasq[:500], (datadiff1/datadiff1.std())[:500], 'c-', )
pyplot.show()
pyplot.clf()
"""
if self.debug is True:
apDisplay.printColor("Noise Model Complete in %s"
%(apDisplay.timeString(time.time()-t0)), "cyan")
return bestfitparams
def start(self):
### new stack path
oldstack = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
newstack = os.path.join(self.params['rundir'], self.stackdata['name'])
apStack.checkForPreviousStack(newstack)
### get particles from stack
apDisplay.printMsg("Querying stack particles")
t0 = time.time()
stackpartq = appiondata.ApRefineParticleData()
stackpartq['refineIter'] = self.iterdata
particles = stackpartq.query()
apDisplay.printMsg("Finished in "+apDisplay.timeString(time.time()-t0))
### write included particles to text file
includeParticle = []
excludeParticle = 0
f = open("test.log", "w")
count = 0
apDisplay.printMsg("Processing stack particles")
t0 = time.time()
for part in particles:
count += 1
if count%500 == 0:
sys.stderr.write(".")
emanstackpartnum = part['particle']['particleNumber']-1
if part['postRefine_keep'] == 1:
### good particle
includeParticle.append(emanstackpartnum)
f.write("%d\t%d\tinclude\n"%(count, emanstackpartnum))
else:
### bad particle
excludeParticle += 1
f.write("%d\t%d\texclude\n"%(count, emanstackpartnum))
sys.stderr.write("\n")
apDisplay.printMsg("Finished in "+apDisplay.timeString(time.time()-t0))
f.close()
includeParticle.sort()
apDisplay.printMsg("Keeping "+str(len(includeParticle))
+" and excluding "+str(excludeParticle)+" particles")
### write kept particles to file
self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list")
apDisplay.printMsg("writing to keepfile "+self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
for partnum in includeParticle:
kf.write(str(partnum)+"\n")
kf.close()
### get number of particles
numparticles = len(includeParticle)
self.params['description'] += ( " ... %d no jumpers substack" % (numparticles,))
### create the new sub stack
apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=True)
if not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
apStack.averageStack(stack=newstack)
if self.params['commit'] is True:
apStack.commitSubStack(self.params)
newstackid = apStack.getStackIdFromPath(newstack)
apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=6)
def crossCorrelateAndShift(infile, reffile, alignfile, ccdocfile, numpart, dataext=".spi"):
### rewriten to do the whole thing in memory in SPIDER, it should be faster
starttime = time.time()
infile = spyder.fileFilter(infile)
reffile = spyder.fileFilter(reffile)
alignfile = spyder.fileFilter(alignfile)
partimg = "_4"
ccmap = "_5"
windccmap = "_6"
boxsize = apFile.getBoxSize(infile+dataext)
if not os.path.isfile(infile+dataext):
apDisplay.printError("input stack file not found: "+infile+dataext)
if not os.path.isfile(reffile+dataext):
apDisplay.printError("reference stack file not found: "+reffile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
### Allocate empty stack
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d"%(boxsize), #boxsize
str(numpart+1), #num part to create in memory
str(numpart+1), #max particle number
)
partnum = 0
while partnum < numpart:
partnum+=1
mySpider.toSpiderQuiet("CP",
infile+("@%05d"%(partnum)), #picture
partimg,
)
### cross correlate images; reversed order to avoid -1*shift
mySpider.toSpiderQuiet("CC N",
reffile+("@%05d"%(partnum)), #reference
partimg, #picture
ccmap, #output file
)
### cannot shift more the 1/4 size of the image
mySpider.toSpiderQuiet("FI x52", partimg, "12" )
mySpider.toSpiderQuiet("x54=int(x52/2)") #window size
mySpider.toSpiderQuiet("x55=int(x52/4)") #window topleft
mySpider.toSpiderQuiet("WI",
ccmap, #input file
windccmap, #output file
"x54,x54", #window size
"x55,x55", #window origin
)
### find the cross-correlation peak
mySpider.toSpiderQuiet("x56=int(x52/4)+1") #center of window
mySpider.toSpiderQuiet("PK M x11,x12,x13,x14",
windccmap, #input ccmap file
"x56,x56", #origin coordinates
)
### save info to doc file
mySpider.toSpiderQuiet("SD %d,x13,x14"%(partnum),
ccdocfile, #input ccmap file
)
### shift the images images
mySpider.toSpiderQuiet("SH",
partimg, #old stack
("_2@%05d"%(partnum)), #new stack
"x13,x14", #shift value file
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP", "_2@",
alignfile+"@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE", "_2",
)
mySpider.close()
apDisplay.printColor("finished shifting particles in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def getStackParticleParams(self):
"""
for each particle in the stack, get the information that RELION needs
"""
stackPartList = apStack.getStackParticlesFromId(self.params['stackid'])
if 'last' not in self.params:
self.params['last'] = len(stackPartList)
firstImageId = stackPartList[0]['particle']['image'].dbid
count = 0
lastImageId = -1
lastCtfData = None
lastKv = -1
partParamsList = []
sys.stderr.write("reading stack particle data\n")
t0 = time.time()
for stackPart in stackPartList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
if count % 10000 == 0:
sys.stderr.write("\nparticle %d of %d\n" %
(count, self.params['last']))
# extra particle number information not read by Relion
if count != stackPart['particleNumber']:
apDisplay.printWarning(
"particle number in database is not in sync")
if count > self.params['last']:
break
partParams = {}
partParams['ptclnum'] = count
partParams['filmNum'] = self.getFilmNumber(stackPart, firstImageId)
#print partParams['filmNum']
### get image data
imagedata = stackPart['particle']['image']
if self.originalStackData.defocpair is True:
imagedata = apDefocalPairs.getDefocusPair(imagedata)
if lastImageId == imagedata.dbid:
ctfdata = lastCtfData
partParams['kv'] = lastKv
else:
ctfdata = ctfdb.getBestCtfValue(
imagedata, msg=False, method=self.params['ctfmethod'])
partParams['kv'] = imagedata['scope']['high tension'] / 1000.0
lastCtfData = ctfdata
lastImageId = imagedata.dbid
lastKv = partParams['kv']
### get CTF data from image
if ctfdata is not None:
# use defocus & astigmatism values
partParams['defocus1'] = abs(ctfdata['defocus1'] * 1e10)
partParams['defocus2'] = abs(ctfdata['defocus2'] * 1e10)
partParams['angle_astigmatism'] = ctfdata['angle_astigmatism']
partParams['amplitude_contrast'] = ctfdata[
'amplitude_contrast']
else:
apDisplay.printWarning(
"No ctf information for particle %d in image %d" %
(count, imagedata.dbid))
partParams['defocus1'] = 0.1
partParams['defocus2'] = 0.1
partParams['angle_astigmatism'] = 0.0
partParams['amplitude_contrast'] = 0.07
if self.params['reconiterid'] is not None:
eulerDict = self.getStackParticleEulersForIteration(stackPart)
partParams.update(eulerDict)
partParamsList.append(partParams)
print "no class %d ; mismatch %d" % (self.noClassification,
self.mismatch)
sys.stderr.write("\ndone in %s\n\n" %
(apDisplay.timeString(time.time() - t0)))
return partParamsList
