def makeNewStack(oldstack, newstack, listfile=None, remove=False, bad=False):
if not os.path.isfile(oldstack):
apDisplay.printWarning("could not find old stack: "+oldstack)
if os.path.isfile(newstack):
if remove is True:
apDisplay.printWarning("removing old stack: "+newstack)
time.sleep(2)
apFile.removeStack(newstack)
else:
apDisplay.printError("new stack already exists: "+newstack)
apDisplay.printMsg("creating a new stack\n\t"+newstack+
"\nfrom the oldstack\n\t"+oldstack+"\nusing list file\n\t"+str(listfile))
emancmd = "proc2d "+oldstack+" "+newstack
if listfile is not None:
emancmd += " list="+listfile
apEMAN.executeEmanCmd(emancmd, verbose=True)
if bad is True and listfile is not None:
### run only if num bad particles < num good particles
newstacknumpart = apFile.numImagesInStack(newstack)
oldstacknumpart = apFile.numImagesInStack(oldstack)
if newstacknumpart > oldstacknumpart/2:
### create bad.hed stack with all bad particles
badstack = os.path.join(os.path.dirname(newstack), "bad.hed")
emancmd = "proc2d %s %s exclude=%s"%(oldstack, badstack, listfile)
apEMAN.executeEmanCmd(emancmd, verbose=True)
else:
apDisplay.printMsg("Rejecting more particles than keeping, not creating a bad stack")
return
def setupInitialModel(self):
modeldata = apModel.getModelFromId(self.params['modelid'])
modelfile = os.path.join(modeldata['path']['path'], modeldata['name'])
modelsym = modeldata['symmetry']['symmetry']
apDisplay.printMsg("Current symmetry: %s"%(modeldata['symmetry']['description']))
# if icosahedral recon, rotate volume to 3DEM standard orientation
if modelsym.startswith('Icos (5 3 2)'):
tempfile = "temp.mrc"
emancmd = 'proc3d %s %s icos5fTo2f' % (modelfile, tempfile)
apEMAN.executeEmanCmd(emancmd, verbose=True)
modelfile = tempfile
if modelsym.startswith('Icos (5 3 2)') or modelsym.startswith('Icos (2 5 3)'):
tempfile = "temp.mrc"
emancmd = 'proc3d %s %s rotspin=0,0,1,90' % (modelfile, tempfile)
apEMAN.executeEmanCmd(emancmd, verbose=True)
modelfile = tempfile
outmodelfile = os.path.join(self.params['rundir'], "initmodel.hed")
apFile.removeStack(outmodelfile, warn=False)
emancmd = "proc3d %s %s clip=%d,%d,%d " % (modelfile, outmodelfile, self.boxsize, self.boxsize, self.boxsize)
# rescale initial model if necessary
scale = modeldata['pixelsize']/self.apix
if abs(scale - 1.0) > 1e-3:
emancmd += "scale=%.4f "%(scale)
apEMAN.executeEmanCmd(emancmd, verbose=True)
apImagicFile.setImagic4DHeader(outmodelfile, machineonly=True)
return outmodelfile
def start(self):
knownstackdata = apStack.getOnlyStackData(self.params['knownstackid'])
fullstackdata = apStack.getOnlyStackData(self.params['fullstackid'])
### get good particle numbers
includeParticle, tiltParticlesData = self.getGoodParticles()
self.numpart = len(includeParticle)
### write kept particles to file
self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile"+self.timestamp+".lst")
apDisplay.printMsg("writing to keepfile "+self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
for partnum in includeParticle:
kf.write(str(partnum)+"\n")
kf.close()
### make new stack of tilted particle from that run
fullstackfile = os.path.join(fullstackdata['path']['path'], fullstackdata['name'])
sb = os.path.splitext(fullstackdata['name'])
newname = "tiltpairsub%d" % self.params['knownstackid']+sb[-1]
newstackfile = os.path.join(self.params['rundir'], newname)
apFile.removeStack(newstackfile, warn=False)
apStack.makeNewStack(fullstackfile, newstackfile, self.params['keepfile'])
if not os.path.isfile(newstackfile):
apDisplay.printError("No stack was created")
self.params['stackid'] = self.params['fullstackid']
apStack.commitSubStack(self.params, newname, sorted=False)
apStack.averageStack(stack=newstackfile)
newstackid = apStack.getStackIdFromPath(newstackfile)
if self.params['meanplot'] is True:
apDisplay.printMsg("creating Stack Mean Plot montage for stackid")
apStackMeanPlot.makeStackMeanPlot(newstackid)
def setupParticles(self):
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
apix = self.stack['apix']*self.params['bin']
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1:
clipsize = boxsize*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
if self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
proccmd += " last="+str(self.params['numpart']-1)
apFile.removeStack(self.params['localstack'], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
return (boxsize, apix)
def setupParticles(self):
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
apix = self.stack['apix']*self.params['bin']
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1:
clipsize = boxsize*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
if self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
proccmd += " last="+str(self.params['numpart']-1)
apFile.removeStack(self.params['localstack'], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
return (boxsize, apix)
def createSplitStacks(self):
self.splitStacks = {}
self.splitStacks['left'] = "particles.left.mrc"
self.splitStacks['right'] = "particles.right.mrc"
if not os.path.isfile(self.splitStacks['left']):
pass
elif not os.path.isfile(self.splitStacks['right']):
pass
elif apFile.fileSize(self.splitStacks['left']) < 100:
pass
elif apFile.fileSize(self.splitStacks['right']) < 100:
pass
else:
apDisplay.printMsg("split stacks already exist, skipping this step")
return
apDisplay.printColor("Step 1A: Splitting input stack %s into two parts"
%(self.params['inputstack']), "purple")
oddfile, evenfile = apImagicFile.splitStackEvenOdd(self.params['inputstack'],
rundir=self.params['rundir'], msg=True)
apDisplay.printColor("Step 1B: Converting odd particles from %s into Left MRC stack %s"
%(oddfile, self.splitStacks['left']), "purple")
imagic2mrc.imagic_to_mrc(oddfile, self.splitStacks['left'])
apFile.removeStack(oddfile)
apDisplay.printColor("Step 1C: Converting even particles from %s into Right MRC stack %s"
%(evenfile, self.splitStacks['right']), "purple")
imagic2mrc.imagic_to_mrc(evenfile, self.splitStacks['right'])
apFile.removeStack(evenfile)
return
def writeToStack(self,partarray):
if self.partperiter == 0:
arrayshape = partarray.shape
partperiter = int(1e9/(arrayshape[0]*arrayshape[1])/16.)
if partperiter > 4096:
partperiter = 4096
self.partperiter = partperiter
apDisplay.printMsg("Using %d particle per iteration"%(partperiter))
stackroot = self.outstackfile[:-4]
imgnum = self.imgnum
index = imgnum % self.partperiter
### Process images
startmem = mem.active()
index = imgnum % self.partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
#sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index))
if mem.active()-startmem > 2e6:
apDisplay.printWarning("Out of memory")
if index < 1:
### deal with large stacks, reset loop
if imgnum > 0:
sys.stderr.write("\n")
stackname = "%s-%d.hed"%(stackroot, imgnum)
apDisplay.printMsg("writing single particles to file "+stackname)
self.stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(self.stackarray, stackname, msg=False)
perpart = (time.time()-self.starttime)/imgnum
apDisplay.printColor("%d :: %.1fM mem :: %s/part "%
(imgnum+1, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart)),
"blue")
self.stackarray = []
### merge particles
self.stackarray.append(partarray)
def setupParticles(self):
self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed")
self.stack = {}
self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"])
self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"])
self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"])
self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"])
boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2
apix = self.stack["apix"] * self.params["bin"]
proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"])
if self.params["bin"] > 1:
clipsize = boxsize * self.params["bin"]
proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize)
if self.params["highpass"] > 1:
proccmd += " hp=" + str(self.params["highpass"])
if self.params["lowpass"] > 1:
proccmd += " lp=" + str(self.params["lowpass"])
proccmd += " last=" + str(self.params["numpart"] - 1)
apFile.removeStack(self.params["localstack"], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"])
return (boxsize, apix)
def createAlignedReferenceStack(self):
searchstr = "part"+self.params['timestamp']+"_ref0*.xmp"
files = glob.glob(searchstr)
files.sort()
stack = []
reflist = self.readRefDocFile()
for i in range(len(files)):
fname = files[i]
refdict = reflist[i]
if refdict['partnum'] != i+1:
print i, refdict['partnum']
apDisplay.printError("sorting error in reflist, see neil")
refarray = spider.read(fname)
xyshift = (refdict['xshift'], refdict['yshift'])
alignrefarray = apImage.xmippTransform(refarray, rot=refdict['inplane'],
shift=xyshift, mirror=refdict['mirror'])
stack.append(alignrefarray)
stackarray = numpy.asarray(stack, dtype=numpy.float32)
#print stackarray.shape
avgstack = "part"+self.params['timestamp']+"_average.hed"
apFile.removeStack(avgstack, warn=False)
apImagicFile.writeImagic(stackarray, avgstack)
### create a average mrc
avgdata = stackarray.mean(0)
apImage.arrayToMrc(avgdata, "average.mrc")
return
def preprocessModelWithProc3d(self, rescale=True):
'''
Use EMAN proc3d to scale initial or mask model to that of the refinestack and format
'''
extname = self.proc3dFormatConversion()
if self.model['ismask'] is False:
modelname = "initmodel"
else:
modelname = "maskvol"
outmodelfile = os.path.join(
self.params['rundir'],
"%s%04d.%s" % (modelname, self.model['id'], extname))
apFile.removeStack(outmodelfile, warn=False)
if rescale:
apVolume.rescaleVolume(self.model['file'],
outmodelfile,
self.model['apix'],
self.stack['apix'],
self.stack['boxsize'],
spider=self.modelspidersingle)
symmetry_description = self.model['data']['symmetry']['symmetry']
if 'Icos' in symmetry_description:
self.convertRefineModelIcosSymmetry(modelname, extname,
outmodelfile,
self.stack['apix'])
self.model['file'] = outmodelfile
self.model['apix'] = self.stack['apix']
self.model['format'] = extname
return outmodelfile
def setupInitialModel(self):
modeldata = apModel.getModelFromId(self.params["modelid"])
modelfile = os.path.join(modeldata["path"]["path"], modeldata["name"])
modelsym = modeldata["symmetry"]["symmetry"]
apDisplay.printMsg("Current symmetry: %s" % (modeldata["symmetry"]["description"]))
# if icosahedral recon, rotate volume to 3DEM standard orientation
if modelsym.startswith("Icos (5 3 2)"):
tempfile = "temp.mrc"
emancmd = "proc3d %s %s icos5fTo2f" % (modelfile, tempfile)
apEMAN.executeEmanCmd(emancmd, verbose=True)
modelfile = tempfile
if modelsym.startswith("Icos (5 3 2)") or modelsym.startswith("Icos (2 5 3)"):
tempfile = "temp.mrc"
emancmd = "proc3d %s %s rotspin=0,0,1,90" % (modelfile, tempfile)
apEMAN.executeEmanCmd(emancmd, verbose=True)
modelfile = tempfile
outmodelfile = os.path.join(self.params["rundir"], "initmodel.hed")
apFile.removeStack(outmodelfile, warn=False)
emancmd = "proc3d %s %s clip=%d,%d,%d " % (modelfile, outmodelfile, self.boxsize, self.boxsize, self.boxsize)
# rescale initial model if necessary
scale = modeldata["pixelsize"] / self.apix
if abs(scale - 1.0) > 1e-3:
emancmd += "scale=%.4f " % (scale)
apEMAN.executeEmanCmd(emancmd, verbose=True)
apImagicFile.setImagic4DHeader(outmodelfile, machineonly=True)
return outmodelfile
def start(self):
### load parameters
runparams = self.readRunParameters()
### align references
self.alignReferences(runparams)
### create an aligned stack
self.createAlignedReferenceStack()
### read particles
self.lastiter = self.findLastIterNumber()
if self.params['sort'] is True:
self.sortFolder()
reflist = self.readRefDocFile()
partlist = self.readPartDocFile(reflist)
self.writePartDocFile(partlist)
### create aligned stacks
alignimagicfile = self.createAlignedStacks(partlist, runparams['localstack'])
apStack.averageStack(alignimagicfile)
### calculate resolution for each reference
apix = apStack.getStackPixelSizeFromStackId(runparams['stackid'])*runparams['bin']
self.calcResolution(partlist, alignimagicfile, apix)
### insert into databse
self.insertRunIntoDatabase(alignimagicfile, runparams)
self.insertParticlesIntoDatabase(runparams['stackid'], partlist)
apFile.removeStack(runparams['localstack'], warn=False)
rmcmd = "/bin/rm -fr partfiles/*"
apEMAN.executeEmanCmd(rmcmd, verbose=False, showcmd=False)
apFile.removeFilePattern("partfiles/*")
def start(self):
"""
this is the main component of the script
where all the processing is done
"""
self.params['notstack'] = os.path.join(
self.params['rundir'], "notstack-" + self.timestamp + ".hed")
self.params['tiltstack'] = os.path.join(
self.params['rundir'], "tiltstack-" + self.timestamp + ".hed")
apFile.removeStack(self.params['notstack'])
apFile.removeStack(self.params['tiltstack'])
parttree = self.getParticles(self.params['stack1'],
self.params['stack2'])
### copy density file
density = self.timestamp + ".mrc"
shutil.copy(self.params['density'], density)
self.generateProjections(parttree, density)
### double check sizes
self.checksizes()
def runEMANmra(self):
# set up cls files
emancmd = "proc2d %s.hed cls mraprep" %self.params['currentcls']
apEMAN.executeEmanCmd(emancmd, verbose=False)
# run EMAN projection matching
emancmd = "classesbymra %s %s.hed mask=%i split imask=-1 logit=1 sep=1 phase" %(
self.params['localstack'],
self.params['currentcls'],
self.workingmask
)
if self.params['nproc'] > 1:
apEMAN.executeRunpar(emancmd, self.params['nproc'])
else:
apEMAN.executeEmanCmd(emancmd, verbose=True)
# create stack of aligned particles
# first get list of cls files
clslist=glob.glob('cls*.lst')
clslist.sort()
emantar = tarfile.open("cls.eman.tar","w")
clsarray = [[]for i in range(self.params['numpart'])]
for cls in clslist:
f = open(cls)
lines = f.readlines()
f.close()
for l in range(1,len(lines)):
d=lines[l].strip().split()
if len(d) < 4:
continue
part = int(d[0])
stack = d[1]
cc = float(d[2][:-1])
(rot,x,y,mirror) = d[3].split(',')
clsarray[part]=[part,stack,cc,float(rot),float(x),float(y),int(mirror)]
# for tarring cls files
emantar.add(cls)
emantar.close()
# remove eman cls####.lst files
for cls in clslist:
os.remove(cls)
# create a new cls file with all particles
f = open("cls_all.lst","w")
f.write("#LST\n")
for p in clsarray:
f.write("%i\t%s\t%.2f, %.6f,%.3f,%.3f,%i\n" % (p[0],p[1],p[2],p[3],p[4],p[5],p[6]))
f.close()
# create aligned particles
apEMAN.alignParticlesInLST("cls_all.lst","mrastack.hed")
if self.params['currentiter'] > 1:
apFile.removeStack(self.params['alignedstack'])
self.params['alignedstack']=os.path.abspath("mrastack")
self.params['alignedstack'] = string.replace(self.params['alignedstack'],"/jetstor/APPION","")
# remove cls file used for alignment
os.remove("cls_all.lst")
def renderSlice(density, box=None, tmpfile=None, sym='c1'):
"""
create mrc of central slice for viruses
"""
if isValidVolume(density) is False:
apDisplay.printError("Volume file is not valid")
if tmpfile is None:
tmpfile = density
if box is None:
boxdims = apFile.getBoxSize(tmpfile)
box = boxdims[0]
halfbox = int(box/2)
tmphed = density + '.hed'
tmpimg = density + '.img'
hedcmd = ('proc3d %s %s' % (tmpfile, tmphed))
if sym.lower()[:4] != 'icos':
hedcmd = hedcmd + " rot=90"
proc = subprocess.Popen(hedcmd, shell=True)
proc.wait()
pngslice = density + '.slice.png'
slicecmd = ('proc2d %s %s first=%i last=%i' % (tmphed, pngslice, halfbox, halfbox))
proc = subprocess.Popen(slicecmd, shell=True)
proc.wait()
apFile.removeStack(tmphed, warn=False)
return
def scaleTemplates(self):
reffile = os.path.join(self.params['rundir'], "references.hed")
if self.params['apix'] != self.templatestack['apix']:
scalefactor = float(
self.templatestack['apix']) / self.params['apix']
templates = apImagicFile.readImagic(reffile)
scaledtemplates = []
for templatearray in templates['images']:
newarray = apTemplate.scaleTemplate(templatearray, scalefactor)
scaledtemplates.append(newarray)
apImagicFile.writeImagic(scaledtemplates, reffile)
refbox = apFile.getBoxSize(reffile)[0]
stbox = self.params['boxsize']
### now clip the references to get identical boxsizes
if stbox != refbox:
while os.path.isfile(reffile + ".new.img"):
apFile.removeStack(reffile + ".new.img")
emancmd = "proc2d " + reffile + " " + reffile + ".new.hed clip=" + str(
stbox) + " edgenorm"
apParam.runCmd(emancmd, "EMAN")
os.rename(reffile + ".new.hed", reffile)
os.rename(reffile + ".new.img", reffile[:-4] + ".img")
return
def renderSlice(density, box=None, tmpfile=None, sym='c1'):
"""
create mrc of central slice for viruses
"""
if isValidVolume(density) is False:
apDisplay.printError("Volume file is not valid")
if tmpfile is None:
tmpfile = density
if box is None:
boxdims = apFile.getBoxSize(tmpfile)
box = boxdims[0]
halfbox = int(box / 2)
tmphed = density + '.hed'
tmpimg = density + '.img'
hedcmd = ('proc3d %s %s' % (tmpfile, tmphed))
if sym.lower()[:4] != 'icos':
hedcmd = hedcmd + " rot=90"
proc = subprocess.Popen(hedcmd, shell=True)
proc.wait()
pngslice = density + '.slice.png'
slicecmd = ('proc2d %s %s first=%i last=%i' %
(tmphed, pngslice, halfbox, halfbox))
proc = subprocess.Popen(slicecmd, shell=True)
proc.wait()
apFile.removeStack(tmphed, warn=False)
return
def start(self):
"""
This is the core of your function.
"""
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp+".hed")
apFile.removeStack(filtered_stackfile, warn=False)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
boxsize = apStack.getStackBoxsize(self.params['stackid'])
emancmd = "proc2d %s %s apix=%.3f "%(original_stackfile, filtered_stackfile, apix)
if self.params['lowpass'] is not None:
emancmd += " lp=%.3f "%(self.params['lowpass'])
if self.params['highpass'] is not None:
emancmd += " hp=%.3f "%(self.params['highpass'])
if self.params['bin'] is not None and self.params['bin'] > 1:
## determine a multiple of the bin that is divisible by 2 and less than the boxsize
clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2*self.params['bin']
emancmd += " shrink=%d clip=%d,%d "%(self.params['bin'], clipsize, clipsize)
emancmd += " last=%d "%(self.params['numpart']-1)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
### confirm that it worked
if self.params['numpart'] != apFile.numImagesInStack(filtered_stackfile):
apDisplay.printError("Missing particles in stack")
### run the radon transform code
self.radonAlign(filtered_stackfile)
### insert info into database
self.commitToDatabase()
def runMaxlike(self):
stackdata = apStack.getOnlyStackData(self.params["stackid"])
apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"])
### process stack to local file
self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed")
proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(apix)
if self.params["highpass"] > 1:
proccmd += " hp=" + str(self.params["highpass"])
if self.params["lowpass"] > 1:
proccmd += " lp=" + str(self.params["lowpass"])
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" "
+ " -i "
+ os.path.join(self.params["rundir"], self.partlistdocfile)
+ " -nref 1 "
+ " -iter 10 "
+ " -o "
+ os.path.join(self.params["rundir"], "part" + self.timestamp)
+ " -fast -C 1e-18 "
)
### angle step
if self.params["rotate"] is True:
xmippopts += " -psi_step 90 "
else:
xmippopts += " -psi_step 360 "
### convergence criteria
if self.params["converge"] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params["converge"] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params["mirror"] is True:
xmippopts += " -mirror "
if self.params["maxshift"] is not None:
xmippopts += " -max_shift %d " % (self.params["maxshift"])
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime))
### create a quick mrc
emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc"
apEMAN.executeEmanCmd(emancmd, verbose=True)
apFile.removeStack(self.params["localstack"])
apFile.removeFilePattern("partfiles/*")
def getClusterStack(self):
"""
get selected class averages from cluster stack
"""
numclusters = self.clusterstackdata['num_classes']
if self.params['excludelist'] is None and self.params[
'includelist'] is None:
### Case 1: Keep all classes
self.params['keepfile'] = None
apDisplay.printMsg("Keeping all %d clusters" % (numclusters))
else:
### Case 2: Keep subset of classes
### list of classes to be excluded
excludelist = []
if self.params['excludelist'] is not None:
excludestrlist = self.params['excludelist'].split(",")
for excludeitem in excludestrlist:
excludelist.append(int(excludeitem.strip()))
apDisplay.printMsg("Exclude list: " + str(excludelist))
### list of classes to be included
includelist = []
if self.params['includelist'] is not None:
includestrlist = self.params['includelist'].split(",")
for includeitem in includestrlist:
includelist.append(int(includeitem.strip()))
apDisplay.printMsg("Include list: " + str(includelist))
### write kept cluster numbers to file
self.params['keepfile'] = os.path.join(
self.params['rundir'], "keepfile-" + self.timestamp + ".list")
apDisplay.printMsg("writing to keepfile " +
self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
count = 0
for clusternum in range(numclusters):
if ((len(includelist) > 0 and clusternum in includelist) or
(len(excludelist) > 0 and not clusternum in excludelist)):
count += 1
kf.write(str(clusternum) + "\n")
kf.close()
apDisplay.printMsg("Keeping %d of %d clusters" %
(count, numclusters))
### override number of clusters with new number
numclusters = count
### create the new sub stack
newstack = os.path.join(self.params['rundir'], "rawclusters.hed")
oldstack = os.path.join(self.clusterstackdata['path']['path'],
self.clusterstackdata['avg_imagicfile'])
apFile.removeStack(newstack)
apStack.makeNewStack(oldstack, newstack, self.params['keepfile'])
if not os.path.isfile(newstack):
apDisplay.printError("No cluster stack was created")
return newstack, numclusters
def runMaxlike(self):
stackdata = apStack.getOnlyStackData(self.params['stackid'])
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
### process stack to local file
self.params['localstack'] = os.path.join(self.params['rundir'],
self.timestamp + ".hed")
proccmd = "proc2d " + stackfile + " " + self.params[
'localstack'] + " apix=" + str(apix)
if self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -nref 1 " + " -iter 10 " + " -o " +
os.path.join(self.params['rundir'], "part" + self.timestamp) +
" -fast -C 1e-18 ")
### angle step
if self.params['rotate'] is True:
xmippopts += " -psi_step 90 "
else:
xmippopts += " -psi_step 360 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
if self.params['maxshift'] is not None:
xmippopts += " -max_shift %d " % (self.params['maxshift'])
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### create a quick mrc
emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc"
apEMAN.executeEmanCmd(emancmd, verbose=True)
apFile.removeStack(self.params['localstack'])
apFile.removeFilePattern("partfiles/*")
def runAffinityPropagation(self, alignedstack):
### Get initial correlation values
### this is really, really slow
similarfile, simlist = self.fillSimilarityMatrix(alignedstack)
### Preference value stats
preffile = self.setPreferences(simlist)
### run apcluster.exe program
outfile = "clusters.out"
apDisplay.printMsg("Run apcluster.exe program")
apclusterexe = os.path.join(apParam.getAppionDirectory(), "bin/apcluster.exe")
apFile.removeFile(outfile)
clustercmd = apclusterexe+" "+similarfile+" "+preffile+" "+outfile
clusttime = time.time()
proc = subprocess.Popen(clustercmd, shell=True)
proc.wait()
apDisplay.printMsg("apCluster time: "+apDisplay.timeString(time.time()-clusttime))
if not os.path.isfile(outfile):
apDisplay.printError("apCluster did not run")
### Parse apcluster output file: clusters.out
apDisplay.printMsg("Parse apcluster output file: "+outfile)
clustf = open(outfile, "r")
### each line is the particle and the number is the class
partnum = 0
classes = {}
for line in clustf:
sline = line.strip()
if sline:
partnum += 1
classnum = int(sline)
if not classnum in classes:
classes[classnum] = [partnum,]
else:
classes[classnum].append(partnum)
clustf.close()
apDisplay.printMsg("Found %d classes"%(len(classes.keys())))
### Create class averages
classavgdata = []
classnames = classes.keys()
classnames.sort()
for classnum in classnames:
apDisplay.printMsg("Class %d, %d members"%(classnum, len(classes[classnum])))
#clsf = open('subcls%04d.lst'%(classnum), 'w')
#for partnum in classes[classnum]:
# clsf.write("%d\n"%(partnum))
#clsf.close()
classdatalist = apImagicFile.readParticleListFromStack(alignedstack, classes[classnum], msg=False)
classdatarray = numpy.asarray(classdatalist)
classavgarray = classdatarray.mean(0)
#mrc.write(classavgarray, 'subcls%04d.mrc'%(classnum))
classavgdata.append(classavgarray)
apFile.removeStack("classaverage-"+self.timestamp+".hed")
apImagicFile.writeImagic(classavgdata, "classaverage-"+self.timestamp+".hed")
return classes
def emanMrcToStack(partlist):
apFile.removeStack("emanmrc.hed", warn=False)
for part in partlist:
mrc.write(part, "temp.mrc")
emancmd = "proc2d temp.mrc emanmrc.hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apFile.removeFile("temp.mrc")
return
def emanSpiderToStack(partlist):
apFile.removeStack("emanspi.hed", warn=False)
for part in partlist:
spider.write(part, "temp.spi")
emancmd = "proc2d temp.spi emanspi.hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apFile.removeFile("temp.spi")
return
def emanMrcToStack(partlist):
apFile.removeStack("emanmrc.hed", warn=False)
for part in partlist:
mrc.write(part, "temp.mrc")
emancmd = "proc2d temp.mrc emanmrc.hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apFile.removeFile("temp.mrc")
return
def emanSpiderToStack(partlist):
apFile.removeStack("emanspi.hed", warn=False)
for part in partlist:
spider.write(part, "temp.spi")
emancmd = "proc2d temp.spi emanspi.hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apFile.removeFile("temp.spi")
return
def addNoise(self, oldstack, noiselevel, SNR): ### create new image with modified SNR basename, extension = os.path.splitext(oldstack) formattedsnr = "%.3f" % (SNR) newstack = basename+"_snr"+formattedsnr+".hed" apFile.removeStack(newstack) emancmd = "proc2d "+oldstack+" "+newstack+" addnoise="+str(noiselevel) apParam.runCmd(emancmd, "EMAN") return newstack
def addNoise(self, oldstack, noiselevel, SNR): ### create new image with modified SNR basename, extension = os.path.splitext(oldstack) formattedsnr = "%.3f" % (SNR) newstack = basename+"_snr"+formattedsnr+".hed" apFile.removeStack(newstack) emancmd = "proc2d "+oldstack+" "+newstack+" addnoise="+str(noiselevel) apParam.runCmd(emancmd, "EMAN") return newstack
def getClusterStack(self):
"""
get selected class averages from cluster stack
"""
numclusters = self.clusterstackdata['num_classes']
if self.params['excludelist'] is None and self.params['includelist'] is None:
### Case 1: Keep all classes
self.params['keepfile'] = None
apDisplay.printMsg("Keeping all %d clusters"%(numclusters))
else:
### Case 2: Keep subset of classes
### list of classes to be excluded
excludelist = []
if self.params['excludelist'] is not None:
excludestrlist = self.params['excludelist'].split(",")
for excludeitem in excludestrlist:
excludelist.append(int(excludeitem.strip()))
apDisplay.printMsg("Exclude list: "+str(excludelist))
### list of classes to be included
includelist = []
if self.params['includelist'] is not None:
includestrlist = self.params['includelist'].split(",")
for includeitem in includestrlist:
includelist.append(int(includeitem.strip()))
apDisplay.printMsg("Include list: "+str(includelist))
### write kept cluster numbers to file
self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list")
apDisplay.printMsg("writing to keepfile "+self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
count = 0
for clusternum in range(numclusters):
if ( (len(includelist) > 0 and clusternum in includelist)
or (len(excludelist) > 0 and not clusternum in excludelist) ):
count+=1
kf.write(str(clusternum)+"\n")
kf.close()
apDisplay.printMsg("Keeping %d of %d clusters"%(count,numclusters))
### override number of clusters with new number
numclusters = count
### create the new sub stack
newstack = os.path.join(self.params['rundir'], "rawclusters.hed")
oldstack = os.path.join(self.clusterstackdata['path']['path'], self.clusterstackdata['avg_imagicfile'])
apFile.removeStack(newstack)
apStack.makeNewStack(oldstack, newstack, self.params['keepfile'])
if not os.path.isfile(newstack):
apDisplay.printError("No cluster stack was created")
return newstack, numclusters
def runIMAGICclassify(self):
bfile = "msaclassify.job"
outfile = "classes"
apFile.removeStack(outfile)
numIters = int(self.params['numpart']*self.params['itermult'])
decrement = self.params['start']-self.params['end']
if self.params['iter']>0:
decrement /= float(self.params['iter'])
numClasses = self.params['start']-(decrement*self.params['currentiter'])
stackfile = self.params['alignedstack']
self.params['currentnumclasses'] = numClasses
f = open(bfile,'w')
f.write("#!/bin/csh -f\n")
f.write("setenv IMAGIC_BATCH 1\n")
f.write("%s/msa/classify.e <<EOF\n" % self.imagicroot)
f.write("IMAGES\n")
f.write("%s\n"%stackfile)
f.write("0\n")
f.write("%i\n"%self.params['activeeigen'])
f.write("YES\n")
f.write("%i\n"%numClasses)
f.write("classes_start\n")
f.write("EOF\n")
f.write("%s/msa/classum.e <<EOF\n" % self.imagicroot)
f.write("%s\n"%stackfile)
f.write("classes_start\n")
f.write("%s\n"%outfile)
f.write("YES\n")
f.write("NONE\n")
f.write("0\n")
if int(self.imagicversion) >= 120619:
f.write("NONE\n") # Mode of summing statistics
f.write("EOF\n")
## make eigenimage stack appion-compatible
f.write("proc2d eigenim.hed eigenim.hed inplace\n")
f.write("touch msaclassify_done.txt\n")
f.write("exit\n")
f.close()
## execute the batch file
aligntime0 = time.time()
apEMAN.executeEmanCmd("chmod 755 "+bfile)
apDisplay.printColor("Running IMAGIC .batch file: %s"%(os.path.abspath(bfile)), "cyan")
apIMAGIC.executeImagicBatchFile(os.path.abspath(bfile))
os.remove("msaclassify_done.txt")
if not os.path.exists(outfile+".hed"):
apDisplay.printError("ERROR IN IMAGIC SUBROUTINE")
apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-aligntime0), "cyan")
def calcResolution(self, level):
self.resdict = {}
D=self.getClassificationAtLevel(level)
for classref in D:
stack=[]
for partnum in D[classref]:
stack.append(apImagicFile.readSingleParticleFromStack("alignedStack.hed",int(partnum)+1,msg=False))
apImagicFile.writeImagic(stack,"tmp.hed")
frcdata = apFourier.spectralSNRStack("tmp.hed", self.apix)
self.resdict[classref] = apFourier.getResolution(frcdata, self.apix, self.boxsize)
apFile.removeStack("tmp.hed")
def processStack(self, stackarray):
tempstackfile = "temp.%03d.hed" % (self.index)
self.stacksToMerge.append(tempstackfile)
#flipping so particles are unchanged
flippedStack = []
for partarray in stackarray:
flippedpartarray = numpy.flipud(partarray)
flippedpartarray = numpy.fliplr(flippedpartarray)
flippedStack.append(flippedpartarray)
apFile.removeStack(tempstackfile, warn=self.msg)
writeImagic(flippedStack, tempstackfile, msg=self.msg)
self.index += len(stackarray)
def wrtieEvenParticles(self, stackfile, outfile=None):
numpart = apFile.numImagesInStack(stackfile)
evenPartList = 2 * numpy.arange(numpart / 2) + 2
self.stacksToMerge = []
if outfile is None:
root = os.path.splitext(stackfile)[0]
outfile = root + ".even.hed"
self.start(stackfile, partlist=evenPartList)
mergeStacks(self.stacksToMerge, outfile, self.msg)
for tempstackfile in self.stacksToMerge:
apFile.removeStack(tempstackfile, warn=self.msg)
return outfile
def calcResolution(self, level):
self.resdict = {}
D=self.getClassificationAtLevel(level)
for classref in D:
stack=[]
for partnum in D[classref]:
stack.append(apImagicFile.readSingleParticleFromStack("alignedStack.hed",int(partnum)+1,msg=False))
apImagicFile.writeImagic(stack,"tmp.hed")
frcdata = apFourier.spectralSNRStack("tmp.hed", self.apix)
self.resdict[classref] = apFourier.getResolution(frcdata, self.apix, self.boxsize)
apFile.removeStack("tmp.hed")
def convertStackToSpider(self, emanstackfile):
"""
takes the stack file and creates a spider file ready for processing
"""
if not os.path.isfile(emanstackfile):
apDisplay.printError("stackfile does not exist: " + emanstackfile)
tempstack = os.path.join(self.params['rundir'],
"filter" + self.timestamp + ".hed")
### first high pass filter particles
apDisplay.printMsg("pre-filtering particles")
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
boxsize = self.getBoxSize()
emancmd = ("proc2d " + emanstackfile + " " + tempstack + " apix=" +
str(apix) + " ")
if self.params['highpasspart'] is not None and self.params[
'highpasspart'] > 0:
emancmd += "hp=" + str(self.params['highpasspart']) + " "
if self.params[
'lowpasspart'] is not None and self.params['lowpasspart'] > 0:
emancmd += "lp=" + str(self.params['lowpasspart']) + " "
if self.params['tiltbin'] > 1:
clipsize = boxsize * self.params['tiltbin']
emancmd += " shrink=%d clip=%d,%d " % (self.params['tiltbin'],
clipsize, clipsize)
apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc2d "
emancmd += tempstack + " "
spiderstack = os.path.join(self.params['rundir'],
"rctstack" + self.timestamp + ".spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack + " "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor(
"Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
time.sleep(1) # wait a sec, for things to finish
apDisplay.printColor(
"finished eman in " +
apDisplay.timeString(time.time() - starttime), "cyan")
apFile.removeStack(tempstack, warn=False)
apFile.removeStack(emanstackfile, warn=False)
if not os.path.isfile(spiderstack):
apDisplay.printError("Failed to create a spider stack")
return spiderstack
def calcResolution(self, level):
self.resdict = {}
D=self.getClassificationAtLevel(level)
for classref in D:
stack=[]
for partnum in D[classref]:
### NOTE: RESOLUTION WILL NOT BE CALCULATED IF ALIGNED STACK IS NOT CREATED
stack.append(apImagicFile.readSingleParticleFromStack(self.params['timestamp']+".hed",int(partnum),msg=False))
apImagicFile.writeImagic(stack,"tmp.hed")
frcdata = apFourier.spectralSNRStack("tmp.hed", self.apix)
self.resdict[classref] = apFourier.getResolution(frcdata, self.apix, self.boxsize)
apFile.removeStack("tmp.hed")
def calcResolution(self, level):
self.resdict = {}
D=self.getClassificationAtLevel(level)
for classref in D:
stack=[]
for partnum in D[classref]:
### NOTE: RESOLUTION WILL NOT BE CALCULATED IF ALIGNED STACK IS NOT CREATED
stack.append(apImagicFile.readSingleParticleFromStack(self.params['timestamp']+".hed",int(partnum),msg=False))
apImagicFile.writeImagic(stack,"tmp.hed")
frcdata = apFourier.spectralSNRStack("tmp.hed", self.apix)
self.resdict[classref] = apFourier.getResolution(frcdata, self.apix, self.boxsize)
apFile.removeStack("tmp.hed")
def applySort(self): out = "classes_avg.hed" # read class averages avgfile = os.path.join(self.params['iterdir'],"classes_avg.img") # make compatible for eman write emancmd = "proc2d %s %s inplace"%(avgfile,avgfile) apEMAN.executeEmanCmd(emancmd,verbose=False) d = EMData.read_images(avgfile) for avgn in self.sortedList: d[avgn].write_image(out,-1) apFile.removeStack(avgfile)
def makeCorrectionStack(oldstackid, oldstack, newstack, listfile=None, remove=False, bad=False):
if not os.path.isfile(oldstack):
apDisplay.printWarning("could not find old stack: "+oldstack)
if os.path.isfile(newstack):
if remove is True:
apDisplay.printWarning("removing old stack: "+newstack)
time.sleep(2)
apFile.removeStack(newstack)
else:
apDisplay.printError("new stack already exists: "+newstack)
apDisplay.printMsg("creating a beam tilt phase correction stack\n\t"+newstack+
"\nfrom the oldstack\n\t"+oldstack+"\nusing list file\n\t"+str(listfile))
correcter = correctStackBeamTiltPhaseShift(oldstackid,newstack)
correcter.start(oldstack)
def calcResolution(self, alignedStack):
self.resdict = {}
for classref, partlist in self.classD.iteritems():
if len(partlist) == 0:
continue
stack=[]
for partnum in partlist:
### NOTE: RESOLUTION WILL NOT BE CALCULATED IF ALIGNED STACK IS NOT CREATED
stack.append(apImagicFile.readSingleParticleFromStack(alignedStack,int(partnum),msg=False))
apImagicFile.writeImagic(stack,"tmp.hed")
frcdata = apFourier.spectralSNRStack("tmp.hed", self.apix)
self.resdict[classref] = apFourier.getResolution(frcdata, self.apix, self.boxsize)
apFile.removeStack("tmp.hed")
def resetStack(self): if self.params['helicalstep'] is not None: ### a new set of ApParticleData are stored, in case ### the inserted particles will be used to create future stacks ### So new selection run name has the helical step size oldsrn = self.selectiondata['name'] newsrn = "%s_%i"%(oldsrn,self.params['helicalstep']) self.newselectiondata = copy.copy(self.selectiondata) self.newselectiondata['name'] = newsrn if self.params['commit'] is True: self.insertStackRun() else: stackfile=os.path.join(self.params['rundir'], self.params['single']) apFile.removeStack(stackfile)
def shift_images(self, filename):
### random shifts and rotations to a stack
shiftstackname = filename[:-4]+"_rand.hed"
apFile.removeStack(shiftstackname)
# shiftfile = os.path.join(self.params['rundir'], "shift_rotate.lst")
shiftfile = os.path.join(self.params['rundir'], "shift.lst")
if os.path.isfile(shiftfile):
apFile.removeFile(shiftfile)
f = open(shiftfile, "a")
# apDisplay.printMsg("Now randomly shifting and rotating particles")
apDisplay.printMsg("Now randomly shifting particles")
for i in range(self.params['projcount']):
# if self.params['rotang'] != 0:
# randrot = random.uniform(-1*self.params['rotang'], self.params['rotang'])
randx = random.uniform(-1*self.params['shiftrad'], self.params['shiftrad'])
randy = random.uniform(-1*self.params['shiftrad'], self.params['shiftrad'])
if self.params['flip'] is True:
flip = random.choice([0,1])
else:
flip = 0
# if self.params['rotang'] != 0:
# emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" rot="+str(randrot)+" trans="+str(randx)+","+str(randy)
# else:
# emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" trans="+str(randx)+","+str(randy)
emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" trans="+str(randx)+","+str(randy)
if flip == 0:
if self.params['pad'] is False:
emancmd = emancmd+" clip="+str(self.params['box'])+","+str(self.params['box'])+" edgenorm"
else:
emancmd = emancmd+" edgenorm"
else:
if self.params['pad'] is False:
emancmd = emancmd+" flip clip="+str(self.params['box'])+","+str(self.params['box'])+" edgenorm"
else:
emancmd = emancmd+" flip edgenorm"
apParam.runCmd(emancmd, "EMAN", showcmd=False)
# if self.params['rotang'] != 0:
# f.write("%.3f,"%(randrot))
# else:
# f.write(",")
f.write("%.3f,"%(randx))
f.write("%.3f,"%(randy))
f.write(str(flip)+"\n")
f.close()
return shiftstackname
def convertStackToImagic(self, stackfile):
### convert spider stack to imagic
emancmd = "proc2d %s "%stackfile
imagicstack = os.path.join(self.params['rundir'], "start.hed")
apFile.removeStack(imagicstack, warn=True)
emancmd += imagicstack+" "
starttime = time.time()
apDisplay.printColor("Running stack conversion, this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
time.sleep(1) # wait a sec, for things to finish
apDisplay.printColor("finished proc2d in "+apDisplay.timeString(time.time()-starttime), "cyan")
if not os.path.isfile(imagicstack):
apDisplay.printError("Failed to create an imagic stack")
return imagicstack
def shift_images(self, filename):
### random shifts and rotations to a stack
shiftstackname = filename[:-4]+"_rand.hed"
apFile.removeStack(shiftstackname)
# shiftfile = os.path.join(self.params['rundir'], "shift_rotate.lst")
shiftfile = os.path.join(self.params['rundir'], "shift.lst")
if os.path.isfile(shiftfile):
apFile.removeFile(shiftfile)
f = open(shiftfile, "a")
# apDisplay.printMsg("Now randomly shifting and rotating particles")
apDisplay.printMsg("Now randomly shifting particles")
for i in range(self.params['projcount']):
# if self.params['rotang'] != 0:
# randrot = random.uniform(-1*self.params['rotang'], self.params['rotang'])
randx = random.uniform(-1*self.params['shiftrad'], self.params['shiftrad'])
randy = random.uniform(-1*self.params['shiftrad'], self.params['shiftrad'])
if self.params['flip'] is True:
flip = random.choice([0,1])
else:
flip = 0
# if self.params['rotang'] != 0:
# emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" rot="+str(randrot)+" trans="+str(randx)+","+str(randy)
# else:
# emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" trans="+str(randx)+","+str(randy)
emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" trans="+str(randx)+","+str(randy)
if flip == 0:
if self.params['pad'] is False:
emancmd = emancmd+" clip="+str(self.params['box'])+","+str(self.params['box'])+" edgenorm"
else:
emancmd = emancmd+" edgenorm"
else:
if self.params['pad'] is False:
emancmd = emancmd+" flip clip="+str(self.params['box'])+","+str(self.params['box'])+" edgenorm"
else:
emancmd = emancmd+" flip edgenorm"
apParam.runCmd(emancmd, "EMAN", showcmd=False)
# if self.params['rotang'] != 0:
# f.write("%.3f,"%(randrot))
# else:
# f.write(",")
f.write("%.3f,"%(randx))
f.write("%.3f,"%(randy))
f.write(str(flip)+"\n")
f.close()
return shiftstackname
def calcResolution(self, alignedStack):
self.resdict = {}
for classref, partlist in self.classD.iteritems():
if len(partlist) == 0:
continue
stack = []
for partnum in partlist:
### NOTE: RESOLUTION WILL NOT BE CALCULATED IF ALIGNED STACK IS NOT CREATED
stack.append(
apImagicFile.readSingleParticleFromStack(alignedStack,
int(partnum),
msg=False))
apImagicFile.writeImagic(stack, "tmp.hed")
frcdata = apFourier.spectralSNRStack("tmp.hed", self.apix)
self.resdict[classref] = apFourier.getResolution(
frcdata, self.apix, self.boxsize)
apFile.removeStack("tmp.hed")
def convertStackToSpider(self, emanstackfile):
"""
takes the stack file and creates a spider file ready for processing
"""
if not os.path.isfile(emanstackfile):
apDisplay.printError("stackfile does not exist: "+emanstackfile)
tempstack = os.path.join(self.params['rundir'], "filter"+self.timestamp+".hed")
### first high pass filter particles
apDisplay.printMsg("pre-filtering particles")
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
boxsize = self.getBoxSize()
emancmd = ("proc2d "+emanstackfile+" "+tempstack
+" apix="+str(apix)+" ")
if self.params['highpasspart'] is not None and self.params['highpasspart'] > 0:
emancmd += "hp="+str(self.params['highpasspart'])+" "
if self.params['lowpasspart'] is not None and self.params['lowpasspart'] > 0:
emancmd += "lp="+str(self.params['lowpasspart'])+" "
if self.params['tiltbin'] > 1:
clipsize = boxsize*self.params['tiltbin']
emancmd += " shrink=%d clip=%d,%d "%(self.params['tiltbin'], clipsize, clipsize)
apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc2d "
emancmd += tempstack+" "
spiderstack = os.path.join(self.params['rundir'], "rctstack"+self.timestamp+".spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
time.sleep(1) # wait a sec, for things to finish
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
apFile.removeStack(tempstack, warn=False)
apFile.removeStack(emanstackfile, warn=False)
if not os.path.isfile(spiderstack):
apDisplay.printError("Failed to create a spider stack")
return spiderstack
def createReferenceStack(self):
avgstack = "part"+self.timestamp+"_average.hed"
apFile.removeStack(avgstack, warn=False)
searchstr = "part"+self.timestamp+"_ref0*.xmp"
files = glob.glob(searchstr)
if len(files) == 0:
apDisplay.printError("Xmipp did not run")
files.sort()
stack = []
for i in range(len(files)):
fname = files[i]
refarray = spider.read(fname)
stack.append(refarray)
apImagicFile.writeImagic(stack, avgstack)
### create a average mrc
stackarray = numpy.asarray(stack, dtype=numpy.float32)
avgdata = stackarray.mean(0)
apImage.arrayToMrc(avgdata, "average.mrc")
return
def makeCorrectionStack(oldstackid,
oldstack,
newstack,
listfile=None,
remove=False,
bad=False):
if not os.path.isfile(oldstack):
apDisplay.printWarning("could not find old stack: " + oldstack)
if os.path.isfile(newstack):
if remove is True:
apDisplay.printWarning("removing old stack: " + newstack)
time.sleep(2)
apFile.removeStack(newstack)
else:
apDisplay.printError("new stack already exists: " + newstack)
apDisplay.printMsg("creating a beam tilt phase correction stack\n\t" +
newstack + "\nfrom the oldstack\n\t" + oldstack +
"\nusing list file\n\t" + str(listfile))
correcter = correctStackBeamTiltPhaseShift(oldstackid, newstack)
correcter.start(oldstack)
def convertStackToImagic(self, stackfile):
### convert spider stack to imagic
emancmd = "proc2d %s " % stackfile
imagicstack = os.path.join(self.params['rundir'], "start.hed")
apFile.removeStack(imagicstack, warn=True)
emancmd += imagicstack + " "
starttime = time.time()
apDisplay.printColor("Running stack conversion, this can take a while",
"cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
time.sleep(1) # wait a sec, for things to finish
apDisplay.printColor(
"finished proc2d in " +
apDisplay.timeString(time.time() - starttime), "cyan")
if not os.path.isfile(imagicstack):
apDisplay.printError("Failed to create an imagic stack")
return imagicstack
def preprocessModelWithProc3d(self, rescale=True): ''' Use EMAN proc3d to scale initial or mask model to that of the refinestack and format ''' extname = self.proc3dFormatConversion() if self.model['ismask'] is False: modelname = "initmodel" else: modelname = "maskvol" outmodelfile = os.path.join(self.params['rundir'], "%s%04d.%s" % (modelname,self.model['id'],extname)) apFile.removeStack(outmodelfile, warn=False) if rescale: apVolume.rescaleVolume(self.model['file'], outmodelfile, self.model['apix'], self.stack['apix'], self.stack['boxsize'], spider=self.modelspidersingle) symmetry_description = self.model['data']['symmetry']['symmetry'] if 'Icos' in symmetry_description: self.convertRefineModelIcosSymmetry(modelname,extname,outmodelfile,self.stack['apix']) self.model['file'] = outmodelfile self.model['apix'] = self.stack['apix'] self.model['format'] = extname return outmodelfile
