def start(self):
# Path of the stack
stackdata = apStack.getOnlyStackData(self.params['stackid'])
fn_oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
# Convert the stack to Xmipp
apXmipp.breakupStackIntoSingleFiles(fn_oldstack)
# Run sort junk
cmd = "xmipp_sort_by_statistics -i partlist.sel"
apDisplay.printColor(cmd, "cyan")
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
# Create sorted stack
apXmipp.gatherSingleFilesIntoStack("sort_junk.sel","sorted.hed")
# Create average MRC
apStack.averageStack("sorted.hed")
# Remove intermediate stuff
#os.unlink("partlist.sel")
#shutil.rmtree("partfiles")
# Upload results
self.uploadResults()
time.sleep(1)
return
def start(self):
# Path of the stack
stackdata = apStack.getOnlyStackData(self.params['stackid'])
fn_oldstack = os.path.join(stackdata['path']['path'],
stackdata['name'])
# Convert the stack to Xmipp
apXmipp.breakupStackIntoSingleFiles(fn_oldstack)
# Run sort junk
cmd = "xmipp_sort_by_statistics -i partlist.sel"
apDisplay.printColor(cmd, "cyan")
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
# Create sorted stack
apXmipp.gatherSingleFilesIntoStack("sort_junk.sel", "sorted.hed")
# Create average MRC
apStack.averageStack("sorted.hed")
# Remove intermediate stuff
#os.unlink("partlist.sel")
#shutil.rmtree("partfiles")
# Upload results
self.uploadResults()
time.sleep(1)
return
def _boxParticlesFromImage(self, imgdata, parttree, imgstackfile):
'''
Box Particles From the manipulated full size image file on disk
'''
### make corrected integrated frame image
imgpath = self.getOriginalImagePath(imgdata)
t0 = time.time()
if self.params['phaseflipped'] is True:
if self.params['fliptype'] == 'emanimage':
### ctf correct whole image using EMAN
imgpath = self.phaseFlipWholeImage(imgpath, imgdata)
elif self.params['fliptype'] == "spiderimage":
imgpath = self.phaseFlipSpider(imgpath, imgdata)
elif self.params['fliptype'][:9] == "ace2image":
### ctf correct whole image using Ace 2
imgpath = self.phaseFlipAceTwo(imgpath, imgdata)
self.ctftimes.append(time.time()-t0)
if imgpath is None:
return None, None, None
### run apBoxer
apDisplay.printMsg("boxing "+str(len(parttree))+" particles into temp file: "+imgstackfile)
### method to align helices
t0 = time.time()
if self.params['rotate'] is True:
apBoxer.boxerRotate(imgpath, parttree, imgstackfile, self.boxsize)
if self.params['finealign'] is True:
from appionlib import apXmipp
apXmipp.breakupStackIntoSingleFiles(imgstackfile, filetype="mrc")
rotcmd = "s_finealign %s %i" %(self.params['rundir'], self.boxsize)
apParam.runCmd(rotcmd, "HIP", verbose=True)
# read in text file containing refined angles
anglepath = os.path.join(self.params['rundir'], 'angles.out')
f = open(anglepath, 'r')
angles = f.readlines()
# loop through parttree and add refined angle to rough angle
for i in range(len(parttree)):
partdict = parttree[i]
fineangle = float(angles[i])
newangle = float(partdict['angle'])-fineangle
partdict['angle'] = newangle
# rerun apBoxer.boxerRotate with the new parttree containing final angles
apBoxer.boxerRotate(imgpath, parttree, imgstackfile, self.boxsize, pixlimit=self.params['pixlimit'])
else:
apBoxer.boxer(imgpath, parttree, imgstackfile, self.boxsize, pixlimit=self.params['pixlimit'])
self.batchboxertimes.append(time.time()-t0)
def runMaxlike(self):
stackdata = apStack.getOnlyStackData(self.params["stackid"])
apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"])
### process stack to local file
self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed")
proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(apix)
if self.params["highpass"] > 1:
proccmd += " hp=" + str(self.params["highpass"])
if self.params["lowpass"] > 1:
proccmd += " lp=" + str(self.params["lowpass"])
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" "
+ " -i "
+ os.path.join(self.params["rundir"], self.partlistdocfile)
+ " -nref 1 "
+ " -iter 10 "
+ " -o "
+ os.path.join(self.params["rundir"], "part" + self.timestamp)
+ " -fast -C 1e-18 "
)
### angle step
if self.params["rotate"] is True:
xmippopts += " -psi_step 90 "
else:
xmippopts += " -psi_step 360 "
### convergence criteria
if self.params["converge"] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params["converge"] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params["mirror"] is True:
xmippopts += " -mirror "
if self.params["maxshift"] is not None:
xmippopts += " -max_shift %d " % (self.params["maxshift"])
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime))
### create a quick mrc
emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc"
apEMAN.executeEmanCmd(emancmd, verbose=True)
apFile.removeStack(self.params["localstack"])
apFile.removeFilePattern("partfiles/*")
def runMaxlike(self):
stackdata = apStack.getOnlyStackData(self.params['stackid'])
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
### process stack to local file
self.params['localstack'] = os.path.join(self.params['rundir'],
self.timestamp + ".hed")
proccmd = "proc2d " + stackfile + " " + self.params[
'localstack'] + " apix=" + str(apix)
if self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -nref 1 " + " -iter 10 " + " -o " +
os.path.join(self.params['rundir'], "part" + self.timestamp) +
" -fast -C 1e-18 ")
### angle step
if self.params['rotate'] is True:
xmippopts += " -psi_step 90 "
else:
xmippopts += " -psi_step 360 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
if self.params['maxshift'] is not None:
xmippopts += " -max_shift %d " % (self.params['maxshift'])
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### create a quick mrc
emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc"
apEMAN.executeEmanCmd(emancmd, verbose=True)
apFile.removeStack(self.params['localstack'])
apFile.removeFilePattern("partfiles/*")
def xmippNormStack(self, inStackPath, outStackPath):
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath)
### setup Xmipp command
xmippexe = apParam.getExecPath("xmipp_normalize", die=True)
apDisplay.printMsg("Using Xmipp to normalize particle stack")
normtime = time.time()
xmippopts = ( " "
+" -i %s"%os.path.join(self.params['rundir'],selfile)
+" -method Ramp "
+" -background circle %i"%(self.stack['boxsize']/self.params['bin']*0.4)
+" -remove_black_dust"
+" -remove_white_dust"
+" -thr_black_dust -%.2f"%(self.params['xmipp-norm'])
+" -thr_white_dust %.2f"%(self.params['xmipp-norm'])
)
xmippcmd = xmippexe+" "+xmippopts
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
normtime = time.time() - normtime
apDisplay.printMsg("Xmipp normalization time: "+apDisplay.timeString(normtime))
### recombine particles to a single imagic stack
tmpstack = "tmp.xmippStack.hed"
apXmipp.gatherSingleFilesIntoStack(selfile,tmpstack)
apFile.moveStack(tmpstack,outStackPath)
### clean up directory
apFile.removeFile(selfile)
apFile.removeDir("partfiles")
def setupParticles(self):
self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed")
self.stack = {}
self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"])
self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"])
self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"])
self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"])
boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2
apix = self.stack["apix"] * self.params["bin"]
proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"])
if self.params["bin"] > 1:
clipsize = boxsize * self.params["bin"]
proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize)
if self.params["highpass"] > 1:
proccmd += " hp=" + str(self.params["highpass"])
if self.params["lowpass"] > 1:
proccmd += " lp=" + str(self.params["lowpass"])
proccmd += " last=" + str(self.params["numpart"] - 1)
apFile.removeStack(self.params["localstack"], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"])
return (boxsize, apix)
def makeStackInFormatFromDefault(stackdata, format='eman'):
'''
Function to create the stack in the requested format
Current default is Imagic format used in EMAN
'''
if format not in formats:
apDisplay.printError('Unknown stack format %s requested' % format)
stackroot, instackdir, outstackdir = getFormattedStackInfo(
stackdata, format)
instackpath = os.path.join(instackdir, stackroot + '.hed')
if format == 'eman':
return stackdata['path']['path']
apParam.createDirectory(outstackdir)
rundir = os.getcwd()
os.chdir(outstackdir)
if format == 'spider':
outstackpath = os.path.join(outstackdir,
stackroot + stack_exts[format][0])
emancmd = "proc2d %s %s spidersingle" % (instackpath, outstackpath)
apEMAN.executeEmanCmd(emancmd, showcmd=True)
if format == 'frealign':
outstackpath = os.path.join(outstackdir,
stackroot + stack_exts[format][0])
emancmd = "proc2d %s %s invert" % (instackpath, outstackpath)
apEMAN.executeEmanCmd(emancmd, showcmd=True)
if format == 'xmipp':
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(instackpath)
os.chdir(rundir)
return outstackdir
def setupParticles(self):
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
apix = self.stack['apix']*self.params['bin']
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1:
clipsize = boxsize*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
if self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
proccmd += " last="+str(self.params['numpart']-1)
apFile.removeStack(self.params['localstack'], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
return (boxsize, apix)
def setupParticles(self):
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
apix = self.stack['apix']*self.params['bin']
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1:
clipsize = boxsize*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
if self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
proccmd += " last="+str(self.params['numpart']-1)
apFile.removeStack(self.params['localstack'], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
return (boxsize, apix)
def xmippNormStack(self, inStackPath, outStackPath):
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath)
### setup Xmipp command
xmippexe = apParam.getExecPath("xmipp_normalize", die=True)
apDisplay.printMsg("Using Xmipp to normalize particle stack")
normtime = time.time()
xmippopts = (" " +
" -i %s" % os.path.join(self.params['rundir'], selfile) +
" -method Ramp " + " -background circle %i" %
(self.stack['boxsize'] / self.params['bin'] * 0.4) +
" -remove_black_dust" + " -remove_white_dust" +
" -thr_black_dust -%.2f" % (self.params['xmipp-norm']) +
" -thr_white_dust %.2f" % (self.params['xmipp-norm']))
xmippcmd = xmippexe + " " + xmippopts
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
normtime = time.time() - normtime
apDisplay.printMsg("Xmipp normalization time: " +
apDisplay.timeString(normtime))
### recombine particles to a single imagic stack
tmpstack = "tmp.xmippStack.hed"
apXmipp.gatherSingleFilesIntoStack(selfile, tmpstack)
apFile.moveStack(tmpstack, outStackPath)
### clean up directory
apFile.removeFile(selfile)
apFile.removeDir("partfiles")
def makeStackInFormatFromDefault(stackdata, format="eman"):
"""
Function to create the stack in the requested format
Current default is Imagic format used in EMAN
"""
if format not in formats:
apDisplay.printError("Unknown stack format %s requested" % format)
stackroot, instackdir, outstackdir = getFormattedStackInfo(stackdata, format)
instackpath = os.path.join(instackdir, stackroot + ".hed")
if format == "eman":
return stackdata["path"]["path"]
apParam.createDirectory(outstackdir)
rundir = os.getcwd()
os.chdir(outstackdir)
if format == "spider":
outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0])
emancmd = "proc2d %s %s spidersingle" % (instackpath, outstackpath)
apEMAN.executeEmanCmd(emancmd, showcmd=True)
if format == "frealign":
outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0])
emancmd = "proc2d %s %s invert" % (instackpath, outstackpath)
apEMAN.executeEmanCmd(emancmd, showcmd=True)
if format == "xmipp":
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(instackpath)
os.chdir(rundir)
return outstackdir
def convertToRefineStack(self):
hedpath = self.stack['file']
selfile = apXmipp.breakupStackIntoSingleFiles(hedpath)
self.stack['file'] = os.path.join(self.params['rundir'], selfile)
self.stack['format'] = 'xmipp'
# remove EMAN Imagic Stack Files
apFile.removeFile(hedpath, warn=True)
imgpath = hedpath[:-3] + 'img'
apFile.removeFile(imgpath, warn=True)
def convertToRefineStack(self):
hedpath = self.stack['file']
selfile = apXmipp.breakupStackIntoSingleFiles(hedpath)
self.stack['file'] = os.path.join(self.params['rundir'],selfile)
self.stack['format'] = 'xmipp'
# remove EMAN Imagic Stack Files
apFile.removeFile(hedpath, warn=True)
imgpath = hedpath[:-3]+'img'
apFile.removeFile(imgpath, warn=True)
def applyEnvelopeAndCTF(self, stack):
### get defocus lists
numpart = self.params['projcount']
cut = int(numpart/80.0)+1
apDisplay.printMsg("%d particles per dot"%(cut))
if len(self.deflist1) == 0:
self.getListOfDefoci(numpart)
### break up particles
partlistdocfile = apXmipp.breakupStackIntoSingleFiles(stack, filetype="mrc")
t0 = time.time()
apDisplay.printMsg("Applying CTF and Envelop to particles")
### apply CTF using ACE2
ctfapplydocfile = self.applyCTFToDocFile(partlistdocfile)
### apply Envelop using ACE2
envelopdocfile = self.applyEnvelopToDocFile(ctfapplydocfile)
### correct CTF using ACE2
if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True:
ctfcorrectdocfile = self.correctCTFToDocFile(envelopdocfile)
else:
ctfcorrectdocfile = envelopdocfile
timeper = (time.time()-t0)/float(numpart)
apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green")
apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green")
### write corrected particle list to doc file
ctfpartlist = []
ctfpartlistfile = os.path.join(self.params['rundir'], "ctfpartlist.lst")
inf = open(ctfcorrectdocfile, 'r')
outf = open(ctfpartlistfile, "w")
for line in inf:
### get filename
filename = line.strip().split()[0]
if not os.path.isfile(filename):
apDisplay.printError("CTF and envelop apply failed")
ctfpartlist.append(filename)
outf.write(filename+"\t1\n")
inf.close()
outf.close()
### merge individual files into a common stack
ctfstack = os.path.join(self.params['rundir'], "ctfstack.hed")
apXmipp.gatherSingleFilesIntoStack(ctfpartlistfile, ctfstack, filetype="mrc")
if self.params['pad'] is True:
emancmd = "proc2d %s %s.clip.hed clip=%d,%d" % (ctfstack, ctfstack[:-4], self.params['box'], self.params['box'])
apParam.runCmd(emancmd, "EMAN")
shutil.move("%s.clip.hed" % ctfstack[:-4], "%s.hed" % ctfstack[:-4])
shutil.move("%s.clip.img" % ctfstack[:-4], "%s.img" % ctfstack[:-4])
return ctfstack, ctfpartlist
def applyEnvelopeAndCTF(self, stack):
### get defocus lists
numpart = self.params['projcount']
cut = int(numpart/80.0)+1
apDisplay.printMsg("%d particles per dot"%(cut))
if len(self.deflist1) == 0:
self.getListOfDefoci(numpart)
### break up particles
partlistdocfile = apXmipp.breakupStackIntoSingleFiles(stack, filetype="mrc")
t0 = time.time()
apDisplay.printMsg("Applying CTF and Envelop to particles")
### apply CTF using ACE2
ctfapplydocfile = self.applyCTFToDocFile(partlistdocfile)
### apply Envelop using ACE2
envelopdocfile = self.applyEnvelopToDocFile(ctfapplydocfile)
### correct CTF using ACE2
if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True:
ctfcorrectdocfile = self.correctCTFToDocFile(envelopdocfile)
else:
ctfcorrectdocfile = envelopdocfile
timeper = (time.time()-t0)/float(numpart)
apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green")
apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green")
### write corrected particle list to doc file
ctfpartlist = []
ctfpartlistfile = os.path.join(self.params['rundir'], "ctfpartlist.lst")
inf = open(ctfcorrectdocfile, 'r')
outf = open(ctfpartlistfile, "w")
for line in inf:
### get filename
filename = line.strip().split()[0]
if not os.path.isfile(filename):
apDisplay.printError("CTF and envelop apply failed")
ctfpartlist.append(filename)
outf.write(filename+"\t1\n")
inf.close()
outf.close()
### merge individual files into a common stack
ctfstack = os.path.join(self.params['rundir'], "ctfstack.hed")
apXmipp.gatherSingleFilesIntoStack(ctfpartlistfile, ctfstack, filetype="mrc")
if self.params['pad'] is True:
emancmd = "proc2d %s %s.clip.hed clip=%d,%d" % (ctfstack, ctfstack[:-4], self.params['box'], self.params['box'])
apParam.runCmd(emancmd, "EMAN")
shutil.move("%s.clip.hed" % ctfstack[:-4], "%s.hed" % ctfstack[:-4])
shutil.move("%s.clip.img" % ctfstack[:-4], "%s.img" % ctfstack[:-4])
return ctfstack, ctfpartlist
def start(self):
self.writeParams()
stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False)
stackpath = os.path.abspath(stackdata["path"]["path"])
stackfile = os.path.join(stackpath, "start.hed")
self.params["localstack"] = os.path.join(stackpath, self.timestamp + ".hed")
proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(self.params["step"])
if self.params["bin"] > 1:
proccmd += " shrink=%d" % int(self.params["bin"])
proccmd += " last=" + str(self.params["numpart"] - 1)
apParam.runCmd(proccmd, "EMAN", verbose=True)
if self.params["numpart"] != apFile.numImagesInStack(self.params["localstack"]):
apDisplay.printError("Missing particles in stack")
apXmipp.breakupStackIntoSingleFiles(self.params["localstack"], filetype="mrc")
if self.params["prehip"] == "yes":
if self.params["rise"] is not None:
self.writeLLBO1()
elif self.params["ll1"] is not None:
self.writeLLBO2()
elif self.params["rise"] is None and self.params["ll1"] is None:
apDisplay.printError(
"You must specify either rise and twist or (1,0) and (0,1) layer lines to run preHIP"
)
apDisplay.printMsg("now running s_prehip to set up input files")
apDisplay.printMsg(
"Make sure the layer line/bessel order assignment is correct. If it is not, you may need to adjust some of the input variables."
)
cmd = "s_prehip"
proc = subprocess.Popen(cmd)
proc.wait()
else:
apDisplay.printMsg("now running s_hip2: Phoelix programs")
cmd = "s_hip2"
proc = subprocess.Popen(cmd)
proc.wait()
# apParam.runCmd(cmd, package="Phoelix")
self.putFilesInStack()
self.insertHipRunData()
self.insertHipParamsData()
self.insertHipIterData()
self.insertHipParticleData()
def start(self):
self.writeParams()
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackpath = os.path.abspath(stackdata['path']['path'])
stackfile = os.path.join(stackpath, "start.hed")
self.params['localstack'] = os.path.join(stackpath, self.timestamp+".hed")
proccmd = "proc2d "+stackfile+" "+self.params['localstack']+" apix="+str(self.params['step'])
if self.params['bin'] > 1:
proccmd += " shrink=%d"%int(self.params['bin'])
proccmd += " last="+str(self.params['numpart']-1)
apParam.runCmd(proccmd, "EMAN", verbose=True)
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
apXmipp.breakupStackIntoSingleFiles(self.params['localstack'], filetype="mrc")
if self.params['prehip'] == "yes":
if self.params['rise'] is not None:
self.writeLLBO1()
elif self.params['ll1'] is not None:
self.writeLLBO2()
elif self.params['rise'] is None and self.params['ll1'] is None:
apDisplay.printError("You must specify either rise and twist or (1,0) and (0,1) layer lines to run preHIP")
apDisplay.printMsg("now running s_prehip to set up input files")
apDisplay.printMsg("Make sure the layer line/bessel order assignment is correct. If it is not, you may need to adjust some of the input variables.")
cmd = "s_prehip"
proc = subprocess.Popen(cmd)
proc.wait()
else:
apDisplay.printMsg("now running s_hip2: Phoelix programs")
cmd = "s_hip2"
proc = subprocess.Popen(cmd)
proc.wait()
#apParam.runCmd(cmd, package="Phoelix")
self.putFilesInStack()
self.insertHipRunData()
self.insertHipParamsData()
self.insertHipIterData()
self.insertHipParticleData()
def runMakeSpectra(self, indata):
"""
From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Makespectra
This program generates a Fourier-Bessel decomposition of each image listed in
a *.sel. Then creates a report ASCII file for each image, with the harmonic
energy percentage as a function of the radius, with extension .SPT.
"""
spectratime = time.time()
apDisplay.printMsg("Running make_spectra")
#logging.debug('Inside make_spectra')
#ROB
tempFileNameforSpectra = "tempFileNameforSpectra.txt"
tmpSelFile = apXmipp.breakupStackIntoSingleFiles(
indata, numpart=self.params['numpart'])
makespectracmd = ("xmipp_make_spectra -i %s -o %s \
-x0 %f -y0 %f -r1 %d -r2 %d -low %d -high %d " % (
tmpSelFile,
tempFileNameforSpectra,
self.xOffset,
self.yOffset,
self.params['spectrainnerradius'],
self.params['spectraouterradius'],
self.params['spectralowharmonic'],
self.params['spectrahighharmonic'],
))
#logging.debug(makespectracmd)
apDisplay.printColor(makespectracmd, "cyan")
proc = subprocess.Popen(makespectracmd,
shell=True,
stdout=subprocess.PIPE)
proc.wait()
apDisplay.printMsg("Spectra calculation complete in %s" %
(apDisplay.timeString(time.time() - spectratime)))
time.sleep(1)
return tempFileNameforSpectra
def runMakeSpectra(self, indata):
"""
From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Makespectra
This program generates a Fourier-Bessel decomposition of each image listed in
a *.sel. Then creates a report ASCII file for each image, with the harmonic
energy percentage as a function of the radius, with extension .SPT.
"""
spectratime = time.time()
apDisplay.printMsg("Running make_spectra")
#logging.debug('Inside make_spectra')
#ROB
tempFileNameforSpectra="tempFileNameforSpectra.txt"
tmpSelFile=apXmipp.breakupStackIntoSingleFiles(indata, numpart=self.params['numpart']);
makespectracmd = ( "xmipp_make_spectra -i %s -o %s \
-x0 %f -y0 %f -r1 %d -r2 %d -low %d -high %d "%
(tmpSelFile,
tempFileNameforSpectra,
self.xOffset,
self.yOffset,
self.params['spectrainnerradius'],
self.params['spectraouterradius'],
self.params['spectralowharmonic'],
self.params['spectrahighharmonic'],
)
)
#logging.debug(makespectracmd)
apDisplay.printColor(makespectracmd, "cyan")
proc = subprocess.Popen(makespectracmd, shell=True,stdout=subprocess.PIPE)
proc.wait()
apDisplay.printMsg("Spectra calculation complete in %s"%(apDisplay.timeString(time.time()-spectratime)))
time.sleep(1)
return tempFileNameforSpectra
def start(self):
'''
OptiMod script
'''
############### define short-hand parameter names ############
refine = True
rundir = self.params['rundir']
box = self.params['boxsize']
rbox = self.params['refineboxsize']
apix = self.params['apix']
rapix = self.params['refineapix']
npart = self.params['numpart']
nvol = self.params['num_volumes']
# sym = self.params['sym']
sym = "c1"
psym = self.params['presumed_sym']
first = self.params['firstimage']
nws = self.params['non_weighted_sequence']
threes = self.params['threes']
lmask = self.params['linmask']
asqfilt = self.params['asqfilt']
anginc = self.params['ang_inc']
keep_ordered = self.params['keep_ordered_num']
hamwin = self.params['ham_win']
useEMAN1 = self.params['useEMAN1']
ipv = self.params['images_per_volume']
lp = self.params['3d_lpfilt']
nref = self.params['nref']
PCA = self.params['PCA']
neigens = self.params['numeigens']
recalc = self.params['recalculate']
preftype = self.params['preftype']
mrad = self.params['mask_radius']
irad = self.params['inner_radius']
orad = self.params['outer_radius']
tnproc = self.params['threadnproc']
nproc = self.params['nproc']
oldavgs = self.params['oldavgs']
avgs = os.path.join(self.params['rundir'], self.params['avgs'])
start = self.params['start_at']
############# copy to working directory ############
if not os.path.isfile(os.path.join(avgs)[:-4]+".hed"):
shutil.copyfile(os.path.join(oldavgs[:-4]+".hed"), avgs[:-4]+".hed")
if not os.path.isfile(os.path.join(avgs)[:-4]+".img"):
shutil.copyfile(os.path.join(oldavgs[:-4]+".img"), avgs[:-4]+".img")
if self.params['ravgs'] is not None:
ravgs = os.path.join(self.params['rundir'], self.params['ravgs'])
if not os.path.isfile(self.params['ravgs'][:-4]+".hed"):
shutil.copy(self.params['oldravgs'][:-4]+".hed",
os.path.join(self.params['ravgs'][:-4]+".hed"))
if not os.path.isfile(self.params['ravgs'][:-4]+".img"):
shutil.copy(self.params['oldravgs'][:-4]+".img",
os.path.join(self.params['ravgs'][:-4]+".img"))
### Euler jumper assessment does not make sense when refine images are different
ejassess = False
else:
ravgs = avgs
ejassess = True
########### scale & prealign class averages, if specified #########
if self.params['scale'] is True:
emancmd = "proc2d %s %s_scaled.img scale=%.3f clip=%i,%i edgenorm" \
% (avgs, avgs[:-4], self.scalefactor, 64, 64)
avgs = avgs[:-4]+"_scaled.img"
if start == "none":
while os.path.isfile(avgs):
apFile.removeStack(avgs)
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printColor("skipping stack scaling", "cyan")
if self.imagicroot is not None and useEMAN1 is False:
apIMAGIC.takeoverHeaders(avgs, npart, box)
if self.imagicroot is not None:
if self.params['prealign'] is True:
if start == "none":
avgs = apIMAGIC.prealignClassAverages(rundir, avgs)
apIMAGIC.checkLogFileForErrors(
os.path.join(rundir, "prealignClassAverages.log"))
### select between EMAN1 and IMAGIC for common lines
if self.imagicroot is not None and useEMAN1 is False:
commonlinesdir = os.path.join(rundir, "angular_reconstitution")
else:
commonlinesdir = os.path.join(rundir, "cross_common_lines")
### make links & calculate CCC matrix for 2D averages
apDisplay.printColor("Calculating similarity matrix", "cyan")
ccc_matrix = apCommonLines.calculate_ccc_matrix_2d(avgs)
clsavgs = os.path.split(avgs)[1][:-4]
if not os.path.isdir(commonlinesdir):
os.mkdir(commonlinesdir)
if os.path.islink(os.path.join(commonlinesdir, clsavgs+".hed")):
os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".hed"))
os.symlink(os.path.join(rundir, clsavgs+".hed"),
os.path.join(commonlinesdir, clsavgs+".hed"))
if os.path.islink(os.path.join(commonlinesdir, clsavgs+".img")):
os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".img"))
os.symlink(os.path.join(rundir, clsavgs+".img"),
os.path.join(commonlinesdir, clsavgs+".img"))
################## create multiple 3d0s #####################
if start == "none":
if self.imagicroot is not None and useEMAN1 is False:
cmdlist = []
seqfile = open(os.path.join(rundir,
"sequences_for_angular_reconstitution.dat"), "w")
apDisplay.printColor("Running multiple raw volume calculations", "cyan")
for i in range(nvol):
sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart,
ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
apCommonLines.check_for_duplicates_in_sequence(sequence)
seqfile.write(str(sequence)+"\n")
### create IMAGIC batch file for each model & append them to be threaded
batchfile = apCommonLines.imagic_batch_file(sequence, i+1, avgs, sym,
asqfilt, lmask, apix, box, anginc, keep_ordered, hamwin, lp,
threes=threes, do_not_remove=False)
proc = subprocess.Popen('chmod 755 '+batchfile, shell=True)
proc.wait()
cmdlist.append(batchfile)
os.chdir(rundir)
seqfile.close()
apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)
### check for errors after execution
for i in range(nvol):
apIMAGIC.checkLogFileForErrors(
os.path.join(commonlinesdir, "3d"+str(i+1)+".log"))
else:
### use EMAN1 cross-common lines
cmdlist = []
seqfile = open(os.path.join(rundir, "sequences_for_cross_common_lines.dat"), "w")
for i in range(nvol):
sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart,
ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
apCommonLines.check_for_duplicates_in_sequence(sequence)
seqfile.write(str(sequence)+"\n")
vol_avgfile = os.path.join(commonlinesdir, "vol_%d_averages.hed" % (i+1))
vol_seqfile = os.path.join(commonlinesdir, "vol_%d_averages.txt" % (i+1))
vsf = open(vol_seqfile, "w")
for j in range(ipv):
vsf.write("%d\n" % sequence[j])
vsf.close()
tmpdir = os.path.join(commonlinesdir, "tmp%d" % (i+1))
fullcmd = "proc2d %s %s list=%s ; " % (avgs, vol_avgfile, vol_seqfile)
fullcmd+= "rm -rf %s ; mkdir %s ; cd %s ; " % (tmpdir, tmpdir, tmpdir)
fullcmd+= "startAny %s sym=c1 proc=1 rounds=2 mask=%d ; " \
% (vol_avgfile, mrad/apix)
fullcmd+= "mv threed.0a.mrc ../threed_%d.mrc ; " % (i+1)
fullcmd+= "mv CCL.hed ../CCL_%d.hed ; " % (i+1)
fullcmd+= "mv CCL.img ../CCL_%d.img ; " % (i+1)
cmdlist.append(fullcmd)
seqfile.close()
apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)
for i in range(nvol):
shutil.rmtree(os.path.join(commonlinesdir, "tmp%d" % (i+1)))
else:
apDisplay.printColor("skipping common lines volume calculations", "cyan")
############### move 3-D models volume directory ################
### create volume directory
volumedir = os.path.join(rundir, "volumes")
if not os.path.isdir(volumedir):
os.mkdir(volumedir)
volumes = {}
cmds = []
if start == "none":
apDisplay.printColor("moving volumes for Xmipp 3-D Maximum Likelihood", "cyan")
for i in range(nvol):
if useEMAN1 is False:
volume1 = os.path.join(commonlinesdir, "3d%d_ordered%d_filt.vol" % (i+1,i+1))
else:
volume1 = os.path.join(commonlinesdir, "threed_%d.mrc" % (i+1))
volume2 = os.path.join(volumedir, "3d%d.vol" % (i+1))
if start == "none":
if useEMAN1 is False:
shutil.move(volume1, volume2)
else:
cmds.append("proc3d %s %s spidersingle" % (volume1, volume2))
volumes[(i+1)] = volume2
apThread.threadCommands(cmds, nproc=tnproc)
#################### align 3-D models ##################
if start == "3D_align" or start == "none":
# if start != "3D_assess" and start != "3D_refine":
### run Maximum Likelihood 3-D alignment & align resulting volumes
apDisplay.printColor("Running Xmipp maximum likelihood 3-D alignment", "cyan")
vol_doc_file, alignref = apCommonLines.xmipp_max_like_3d_align(
volumes, nproc, tnproc, nref)
alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)
apDisplay.printColor("Aligning volumes based on 3-D ML parameters", "cyan")
apCommonLines.align_volumes(alignparams, alignref, nvol, tnproc, apix)
else:
apDisplay.printColor("skipping 3D alignment", "cyan")
vol_doc_file, alignref = apCommonLines.findAlignmentParams(
os.path.join(rundir,"max_like_alignment"), nref)
alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)
##################### Principal Component Analysis ##############
apDisplay.printColor("Calculating inter-volume similarity", "cyan")
aligned_volumes = {}
for i in range(nvol):
aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.vol" % (i+1))
# aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.mrc" % (i+1))
if PCA is True:
simfile, sim_matrix = apCommonLines.runPrincipalComponentAnalysis(
aligned_volumes, nvol, neigens, box, apix, recalculate=recalc)
else:
simfile, sim_matrix = apCommonLines.calculate_ccc_matrix_3d(
aligned_volumes, nvol)
################ 3-D affinity propagation #############
### 3-D Affinity Propagation
apDisplay.printColor("Averaging volumes with Affinity Propagation", "cyan")
preffile = apCommonLines.set_preferences(sim_matrix, preftype, nvol)
classes = apCommonLines.run_affinity_propagation(aligned_volumes, simfile,
preffile, box, apix)
######### refine volumes using Xmipp projection matching #########
if start == "3D_align" or start == "3D_refine" or start == "none":
# if start != "3D_assess":
if not os.path.isdir("refinement"):
os.mkdir("refinement")
os.chdir("refinement")
apXmipp.breakupStackIntoSingleFiles(ravgs)
xmippcmd = "xmipp_normalize -i partlist.sel -method OldXmipp"
apParam.runCmd(xmippcmd, "Xmipp")
for i in classes.keys():
emancmd = "proc3d %s %s scale=%.3f clip=%d,%d,%d mask=%s spidersingle" \
% (os.path.join(rundir, "%d.mrc" % i), "%d.vol" % i, \
(1/self.scalefactor), rbox, rbox, rbox, (mrad / rapix))
apParam.runCmd(emancmd, "EMAN")
apCommonLines.refine_volume(rundir, ravgs, i, mrad, irad, orad, rapix, nproc)
emancmd = "proc3d %s %s apix=%.3f" \
% (os.path.join("refine_%d" % i, "3d%d_refined.vol" % i), \
os.path.join(rundir, "%d_r.mrc" % i), rapix)
apParam.runCmd(emancmd, "EMAN")
os.chdir(rundir)
else:
apDisplay.printColor("skipping 3D refinement", "cyan")
################# model evaluation ################
if start == "3D_align" or start == "3D_refine" or start == "3D_assess" or start == "none":
# if start == "none" or start=="3D_assess":
### final model assessment
try:
euler_array = apCommonLines.getEulerValuesForModels(alignparams,
rundir, nvol, npart, threes=threes)
except:
euler_array = None
ejassess = False
if refine is True:
apCommonLines.assess_3Dclass_quality2(rundir, aligned_volumes, sim_matrix,
classes, euler_array, box, rbox, rapix, ravgs, psym, npart, ejassess=ejassess)
else:
apCommonLines.assess_3Dclass_quality(rundir, aligned_volumes, sim_matrix,
classes, euler_array, box, apix, avgs, psym, npart, ejassess=ejassess)
apCommonLines.combineMetrics("final_model_stats.dat",
"final_model_stats_sorted_by_Rcrit.dat", **{"CCPR":(1,1)})
################## upload & cleanup ##################
### upload to database, if specified
self.upload()
### make chimera snapshots
if refine is True:
for i in classes.keys():
if self.params['mass'] is not None:
apChimera.filterAndChimera(
os.path.join(self.params['rundir'], "%d_r.mrc" % i),
res=self.params['3d_lpfilt'],
apix=self.params['refineapix'],
box=self.params['refineboxsize'],
chimtype="snapshot",
contour=2,
zoom=1,
sym="c1",
color="gold",
mass=self.params['mass']
)
### cleanup
snapshots = glob.glob("*.png")
mtlfiles = glob.glob("*.mtl")
objfiles = glob.glob("*.obj")
pyfiles = glob.glob("*mrc.py")
for file in mtlfiles:
os.remove(file)
for file in objfiles:
os.remove(file)
for file in pyfiles:
os.remove(file)
if not os.path.isdir("snapshots"):
os.mkdir("snapshots")
for s in snapshots:
shutil.move(s, os.path.join("snapshots", s))
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local file
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
a.setValue('append',False)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert') is True
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
# clip not yet implemented
# self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### stack is needed by uploadMaxlike.py
#apFile.removeStack(self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -nref "+str(self.params['numrefs'])
+" -iter "+str(self.params['maxiter'])
+" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp)
+" -psi_step "+str(self.params['psistep'])
)
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-17 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### use student's T distribution
if self.params['student'] is True:
xmippopts += " -student "
### write cluster job file
if self.params['cluster'] is True:
self.writeClusterJobFile()
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
self.estimateIterTime()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using "+str(nproc)+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun+" -np "+str(nproc)+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe+" "+xmippopts
self.writeXmippLog(xmippcmd)
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apParam.runCmd(proccmd, "EMAN", verbose=True)
if self.params['numpart'] != apFile.numImagesInStack(
self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -codes " + str(self.params['numrefs']) + " -iter " +
str(self.params['maxiter']) + " -o " + os.path.join(
self.params['rundir'], "part" + self.params['timestamp']))
if self.params['fast']:
xmippopts += " -fast "
if self.params['correlation']:
xmippopts += " -useCorrelation "
if self.params['classical']:
xmippopts += " -classicalMultiref "
if self.params['align']:
xmippopts += " -alignImages "
### use multi-processor command
apDisplay.printColor(
"Using " + str(self.params['nproc']) + " processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True)
mpiruncmd = self.mpirun + " -np " + str(
self.params['nproc']) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd,
package="Xmipp",
verbose=True,
showcmd=True,
logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(self.params['runtime']))
### minor post-processing
self.createReferenceStack()
self.parseOutput()
self.clearIntermediateFiles()
# self.readyUploadFlag()
apParam.dumpParameters(
self.params, "cl2d-" + self.params['timestamp'] + "-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-" +
self.params['timestamp'] +
"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(
self.runparams['stackid']) * self.runparams['bin']
self.Nlevels = len(
glob.glob("part" + self.params['timestamp'] + "_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part" + self.params[
'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels - 1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part" + self.params['timestamp'] +
"_level_%02d_.hed" % level, classnum + 1,
partdict[classnum], len(partdict))
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local temp file
proccmd = "proc2d " + self.stack['file'] + " temp.hed apix=" + str(
self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### process stack to final file
self.params['localstack'] = os.path.join(self.params['rundir'],
self.timestamp + ".hed")
proccmd = "proc2d temp.hed " + self.params[
'localstack'] + " apix=" + str(
self.stack['apix'] * self.params['bin'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
apFile.removeStack("temp.hed")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### convert stack into single spider files
templateselfile = self.initializeTemplates()
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -iter " + str(self.params['maxiter']) + " -ref " +
templateselfile + " -o " +
os.path.join(self.params['rundir'], "part" + self.timestamp) +
" -psi_step " + str(self.params['psistep']))
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-18 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using " + str(nproc - 1) + " processors!",
"green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun + " -np " + str(
nproc - 1) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -codes "+str(self.params['numrefs'])
+" -iter "+str(self.params['maxiter'])
+" -o "+os.path.join(self.params['rundir'], "part"+self.params['timestamp'])
)
if self.params['fast']:
xmippopts += " -fast "
if self.params['correlation']:
xmippopts += " -useCorrelation "
if self.params['classical']:
xmippopts += " -classicalMultiref "
if self.params['align']:
xmippopts += " -alignImages "
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### minor post-processing
self.createReferenceStack()
self.parseOutput()
self.clearIntermediateFiles()
# self.readyUploadFlag()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local temp file
proccmd = "proc2d "+self.stack['file']+" temp.hed apix="+str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize)*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
proccmd += " last="+str(self.params['numpart']-1)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### process stack to final file
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
proccmd = "proc2d temp.hed "+self.params['localstack']+" apix="+str(self.stack['apix']*self.params['bin'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
apFile.removeStack("temp.hed")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### convert stack into single spider files
templateselfile = self.initializeTemplates()
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -iter "+str(self.params['maxiter'])
+" -ref "+templateselfile
+" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp)
+" -psi_step "+str(self.params['psistep'])
)
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-18 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using "+str(nproc-1)+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun+" -np "+str(nproc-1)+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe+" "+xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
