def start(self):
knownstackdata = apStack.getOnlyStackData(self.params['knownstackid'])
fullstackdata = apStack.getOnlyStackData(self.params['fullstackid'])
### get good particle numbers
includeParticle, tiltParticlesData = self.getGoodParticles()
self.numpart = len(includeParticle)
### write kept particles to file
self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile"+self.timestamp+".lst")
apDisplay.printMsg("writing to keepfile "+self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
for partnum in includeParticle:
kf.write(str(partnum)+"\n")
kf.close()
### make new stack of tilted particle from that run
fullstackfile = os.path.join(fullstackdata['path']['path'], fullstackdata['name'])
sb = os.path.splitext(fullstackdata['name'])
newname = "tiltpairsub%d" % self.params['knownstackid']+sb[-1]
newstackfile = os.path.join(self.params['rundir'], newname)
apFile.removeStack(newstackfile, warn=False)
apStack.makeNewStack(fullstackfile, newstackfile, self.params['keepfile'])
if not os.path.isfile(newstackfile):
apDisplay.printError("No stack was created")
self.params['stackid'] = self.params['fullstackid']
apStack.commitSubStack(self.params, newname, sorted=False)
apStack.averageStack(stack=newstackfile)
newstackid = apStack.getStackIdFromPath(newstackfile)
if self.params['meanplot'] is True:
apDisplay.printMsg("creating Stack Mean Plot montage for stackid")
apStackMeanPlot.makeStackMeanPlot(newstackid)
def start(self):
self.iflag = 1
self.setIBLOW()
### get stack info
self.stackdata = apStack.getOnlyStackData(self.params["stackid"])
self.refinestackfile = os.path.join(self.stackdata["path"]["path"], self.stackdata["name"])
apImagicFile.setImagic4DHeader(self.refinestackfile)
### copy stack or start job file
if self.params["cluster"] is False:
# create alias to stack data
pass
if self.params["cluster"] is True:
self.setupMultiNode()
if self.params["reconstackid"] is not None:
self.reconstackdata = apStack.getOnlyStackData(self.params["stackid"])
self.reconstackfile = os.path.join(self.stackdata["path"]["path"], self.stackdata["name"])
else:
self.reconstackfile = self.refinestackfile
### create initial model file
self.currentvol = os.path.basename(self.setupInitialModel())
### create parameter file
self.currentparam = os.path.basename(self.setupParticleParams())
apDisplay.printColor(
"Initial files:\n Stack: %s\n Volume: %s\n Params: %s\n"
% (os.path.basename(self.refinestackfile), self.currentvol, self.currentparam),
"violet",
)
## run frealign for number for refinement cycles
for i in range(self.params["numiter"]):
iternum = i + 1
if self.params["cluster"] is True:
self.multiNodeRun(iternum)
else:
self.singleNodeRun(iternum)
time.sleep(2)
### calculate FSC
# emancmd = 'proc3d %s %s fsc=fsc.eotest.%d' % (evenvol, oddvol, self.params['iter'])
# apEMAN.executeEmanCmd(emancmd, verbose=True)
if self.params["cluster"] is True:
self.prepareForCluster()
print "Done!"
def setRunDir(self):
"""
this function only runs if no rundir is defined at the command line
"""
if self.params['rundir'] is None:
if ('sessionname' in self.params and self.params['sessionname'] is not None ):
# command line users may use sessionname rather than expId
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']])
else:
if ('expId' in self.params and self.params['expId']):
# expId should always be included from appionwrapper derived appionscript
sessiondata = apDatabase.getSessionDataFromSessionId(self.params['expId'])
self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']])
# The rest should not be needed with appionwrapper format
from appionlib import apStack
if ( self.params['rundir'] is None
and 'reconid' in self.params
and self.params['reconid'] is not None ):
self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid'], msg=False)
if ( self.params['rundir'] is None
and 'stackid' in self.params
and self.params['stackid'] is not None ):
#auto set the run directory
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
path = os.path.abspath(stackdata['path']['path'])
path = os.path.dirname(path)
path = os.path.dirname(path)
self.params['rundir'] = os.path.join(path, self.processdirname, self.params['runname'])
self.params['outdir'] = self.params['rundir']
def setupParticles(self):
self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed")
self.stack = {}
self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"])
self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"])
self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"])
self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"])
boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2
apix = self.stack["apix"] * self.params["bin"]
proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"])
if self.params["bin"] > 1:
clipsize = boxsize * self.params["bin"]
proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize)
if self.params["highpass"] > 1:
proccmd += " hp=" + str(self.params["highpass"])
if self.params["lowpass"] > 1:
proccmd += " lp=" + str(self.params["lowpass"])
proccmd += " last=" + str(self.params["numpart"] - 1)
apFile.removeStack(self.params["localstack"], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"])
return (boxsize, apix)
def start(self):
"""
This is the core of your function.
"""
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp+".hed")
apFile.removeStack(filtered_stackfile, warn=False)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
boxsize = apStack.getStackBoxsize(self.params['stackid'])
emancmd = "proc2d %s %s apix=%.3f "%(original_stackfile, filtered_stackfile, apix)
if self.params['lowpass'] is not None:
emancmd += " lp=%.3f "%(self.params['lowpass'])
if self.params['highpass'] is not None:
emancmd += " hp=%.3f "%(self.params['highpass'])
if self.params['bin'] is not None and self.params['bin'] > 1:
## determine a multiple of the bin that is divisible by 2 and less than the boxsize
clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2*self.params['bin']
emancmd += " shrink=%d clip=%d,%d "%(self.params['bin'], clipsize, clipsize)
emancmd += " last=%d "%(self.params['numpart']-1)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
### confirm that it worked
if self.params['numpart'] != apFile.numImagesInStack(filtered_stackfile):
apDisplay.printError("Missing particles in stack")
### run the radon transform code
self.radonAlign(filtered_stackfile)
### insert info into database
self.commitToDatabase()
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['description'] is None:
apDisplay.printError("run description was not defined")
if self.params['templatelist'] is None:
apDisplay.printError("template list was not provided")
if self.params['lastring'] is None:
apDisplay.printError("a last ring radius was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])/self.params['bin']
if self.params['lastring'] > boxsize/2-2:
apDisplay.printError("last ring radius is too big for boxsize "
+str(self.params['lastring'])+" > "+str(boxsize/2-2))
if self.params['lastring']+self.params['xysearch'] > boxsize/2-2:
apDisplay.printError("last ring plus xysearch radius is too big for boxsize "
+str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2))
### convert / check template data
self.templatelist = self.params['templatelist'].strip().split(",")
if not self.templatelist or type(self.templatelist) != type([]):
apDisplay.printError("could not parse template list="+self.params['templatelist'])
self.params['numtemplate'] = len(self.templatelist)
apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates")
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['numrefs'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
self.mpirun = self.checkMPI()
if self.mpirun is None:
apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
if self.params['nproc'] < 2:
apDisplay.printError("Only the MPI version of CL2D is currently supported, must run with > 1 CPU")
def start(self):
### new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
### make sure that old stack is numbered
alignedstack = os.path.join(self.params['rundir'], 'alignstack.hed')
apStack.checkForPreviousStack(alignedstack)
### run centering algorithm
self.runMaxlike()
### create aligned stacks
partlist = self.readPartDocFile()
stackfile = self.createAlignedStacks(partlist)
if not os.path.isfile(alignedstack):
apDisplay.printError("No stack was created")
### get number of particles
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
self.writeFakeKeepFile(numpart)
self.params['description'] += (
" ... %d maxlike centered substack id %d"
% (numpart, self.params['stackid']))
apStack.commitSubStack(self.params, newname='alignstack.hed', centered=True)
apStack.averageStack(stack=alignedstack)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['start'] is None:
apDisplay.printError("a number of starting classes was not provided")
if self.params['end'] is None:
apDisplay.printError("a number of ending classes was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
# check for virtual stack
self.params['virtualdata'] = None
if not os.path.isfile(stackfile):
vstackdata = apStack.getVirtualStackParticlesFromId(self.params['stackid'])
npart = len(vstackdata['particles'])
self.params['virtualdata'] = vstackdata
else:
npart = apFile.numImagesInStack(stackfile)
if self.params['numpart'] is None:
self.params['numpart'] = npart
elif self.params['numpart'] > npart:
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.workingboxsize = math.floor(self.boxsize/self.params['bin'])
if not self.params['mask']:
self.params['mask'] = (self.boxsize/2)-2
self.workingmask = math.floor(self.params['mask']/self.params['bin'])
if self.params['mramethod'] == 'imagic':
self.imagicroot = apIMAGIC.checkImagicExecutablePath()
self.imagicversion = apIMAGIC.getImagicVersion(self.imagicroot)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['rad'] is None:
apDisplay.printError("no particle radius set")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['lastring'] is None:
apDisplay.printError("a last ring radius was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if self.params['incr'] is None:
apDisplay.printError("angular increments are not specified")
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
numstackp = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > numstackp:
apDisplay.printError("trying to use more particles "+str(numstackp)
+" than available "+str(numstackp))
elif self.params['numpart'] == 0:
apDisplay.printWarning("using all "+str(numstackp)+" particles")
self.params['numpart'] = numstackp
boxsize = apStack.getStackBoxsize(self.params['stackid'])
if self.params['lastring'] > boxsize/2-2:
apDisplay.printError("last ring radius is too big for boxsize "
+str(self.params['lastring'])+" > "+str(boxsize/2-2))
if self.params['lastring']+self.params['xysearch'] > boxsize/2-2:
apDisplay.printError("last ring plus xysearch radius is too big for boxsize "
+str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2))
if (self.params['xysearch'] % self.params['xystep']) > 0:
apDisplay.printError("translational search (xy-search) must be divisible by search step (xy-step)")
def start(self):
### new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
### checks
centerstack = os.path.join(self.params['rundir'], 'align.img')
badstack = os.path.join(self.params['rundir'], 'bad.img')
apStack.checkForPreviousStack(centerstack)
### run centering algorithm
keeplist = self.centerParticles(oldstack, centerstack, badstack)
if not os.path.isfile(centerstack):
apDisplay.printError("No stack was created")
self.params['keepfile'] = os.path.join(self.params['rundir'], 'keepfile.txt')
### get number of particles
self.params['description'] += (
(" ... %d eman centered substack id %d"
% (numparticles, self.params['stackid']))
)
apStack.commitSubStack(self.params, newname=os.path.basename(centerstack), centered=True)
apStack.averageStack(stack=centerstack)
if os.path.isfile(badstack):
apStack.averageStack(stack=badstack, outfile='badaverage.mrc')
def runMaxlike(self):
stackdata = apStack.getOnlyStackData(self.params["stackid"])
apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"])
### process stack to local file
self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed")
proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(apix)
if self.params["highpass"] > 1:
proccmd += " hp=" + str(self.params["highpass"])
if self.params["lowpass"] > 1:
proccmd += " lp=" + str(self.params["lowpass"])
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" "
+ " -i "
+ os.path.join(self.params["rundir"], self.partlistdocfile)
+ " -nref 1 "
+ " -iter 10 "
+ " -o "
+ os.path.join(self.params["rundir"], "part" + self.timestamp)
+ " -fast -C 1e-18 "
)
### angle step
if self.params["rotate"] is True:
xmippopts += " -psi_step 90 "
else:
xmippopts += " -psi_step 360 "
### convergence criteria
if self.params["converge"] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params["converge"] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params["mirror"] is True:
xmippopts += " -mirror "
if self.params["maxshift"] is not None:
xmippopts += " -max_shift %d " % (self.params["maxshift"])
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime))
### create a quick mrc
emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc"
apEMAN.executeEmanCmd(emancmd, verbose=True)
apFile.removeStack(self.params["localstack"])
apFile.removeFilePattern("partfiles/*")
def initializeRefinementUploadVariables(self):
""" untars results, reads required parameters, establishes necessary objects for upload script """
apDisplay.printColor("uploading refinement results for %s reconstruction routine" % (self.package), "cyan")
### establish directories and make an appion results directory
self.basepath = os.path.abspath(self.params["rundir"])
self.resultspath = os.path.abspath(os.path.join(self.params["rundir"], str(self.package) + "_results"))
if not os.path.isdir(self.resultspath):
os.mkdir(self.resultspath)
self.reconpath = os.path.abspath(
os.path.join(self.params["rundir"], self.runparams["reconstruction_working_dir"])
)
### get all stack parameters, map particles in reconstruction to particles in stack, get all model data
self.stackdata = apStack.getOnlyStackData(self.runparams["stackid"])
self.stackmapping = apRecon.partnum2defid(self.runparams["stackid"])
self.modeldata = []
if len(self.runparams["modelid"].split(",")) > 1:
models = self.runparams["modelid"].split(",")
for i in range(len(models)):
self.modeldata.append(appiondata.ApInitialModelData.direct_query(int(models[i])))
else:
self.modeldata.append(appiondata.ApInitialModelData.direct_query(self.runparams["modelid"]))
return
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.checkNoRefRun()
### convert stack to spider
spiderstack = self.createSpiderFile()
### create initialization template
if self.params['initmethod'] == 'allaverage':
templatefile = self.averageTemplate()
elif self.params['initmethod'] == 'selectrand':
templatefile = self.selectRandomParticles()
elif self.params['initmethod'] == 'randpart':
templatefile = self.pickRandomParticle()
elif self.params['initmethod'] == 'template':
templatefile = self.getTemplate()
else:
apDisplay.printError("unknown initialization method defined: "
+str(self.params['initmethod'])+" not in "+str(self.initmethods))
apDisplay.printColor("Running spider this can take awhile","cyan")
### run the alignment
aligntime = time.time()
pixrad = int(round(self.params['partrad']/self.stack['apix']/self.params['bin']))
alignedstack, self.partlist = alignment.refFreeAlignParticles(
spiderstack, templatefile,
self.params['numpart'], pixrad,
self.params['firstring'], self.params['lastring'],
rundir = ".")
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=False)
### convert stack to imagic
imagicstack = self.convertSpiderStack(alignedstack)
apFile.removeFile(alignedstack)
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = aligntime
self.insertNoRefRun(imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def start(self):
# Path of the stack
stackdata = apStack.getOnlyStackData(self.params['stackid'])
fn_oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
# Convert the stack to Xmipp
apXmipp.breakupStackIntoSingleFiles(fn_oldstack)
# Run sort junk
cmd = "xmipp_sort_by_statistics -i partlist.sel"
apDisplay.printColor(cmd, "cyan")
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
# Create sorted stack
apXmipp.gatherSingleFilesIntoStack("sort_junk.sel","sorted.hed")
# Create average MRC
apStack.averageStack("sorted.hed")
# Remove intermediate stuff
#os.unlink("partlist.sel")
#shutil.rmtree("partfiles")
# Upload results
self.uploadResults()
time.sleep(1)
return
def checkConflicts(self):
if self.params["stackid"] is None:
apDisplay.printError("stack id was not defined")
if self.params["description"] is None:
apDisplay.printError("run description was not defined")
if self.params["runname"] is None:
apDisplay.printError("run name was not defined")
maxparticles = 150000
if self.params["numpart"] > maxparticles:
apDisplay.printError(
"too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params["numpart"])
)
if self.params["initmethod"] not in self.initmethods:
apDisplay.printError(
"unknown initialization method defined: "
+ str(self.params["initmethod"])
+ " not in "
+ str(self.initmethods)
)
stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False)
stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"])
if self.params["numpart"] > apFile.numImagesInStack(stackfile):
apDisplay.printError(
"trying to use more particles "
+ str(self.params["numpart"])
+ " than available "
+ str(apFile.numImagesInStack(stackfile))
)
def start(self):
### universal particle counter
self.partnum = 1
### final stack file
self.combinefile = os.path.join( self.params['rundir'], self.params['stackfilename'] )
if os.path.isfile(self.combinefile):
apDisplay.printError("A stack with name "+self.params['stackfilename']+" and path "
+self.params['rundir']+" already exists.")
### loop through stacks
for stackstr in self.params['stackids']:
stackid = int(stackstr)
### get stack data
stackdata = apStack.getOnlyStackData(stackid)
### append particle to stack file
self.appendToStack(stackdata)
if self.params['commit'] is True:
### insert stack data
apDisplay.printColor("inserting new stack particles from stackid="+str(stackid), "cyan")
self.commitStack(stackid)
else:
apDisplay.printWarning("not committing data to database")
apStack.averageStack(stack=self.combinefile)
def setupParticles(self):
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
apix = self.stack['apix']*self.params['bin']
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1:
clipsize = boxsize*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
if self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
proccmd += " last="+str(self.params['numpart']-1)
apFile.removeStack(self.params['localstack'], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
return (boxsize, apix)
def insertAlignStackRunIntoDatabase(self, alignimagicfile):
apDisplay.printMsg("Inserting CL2D Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.runparams['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
# if uniquerun:
# apDisplay.printError("Run name '"+self.runparams['runname']+"' and path already exist in database")
### setup cl2d run
cl2dq = appiondata.ApCL2DRunData()
cl2dq['runname'] = self.runparams['runname']
cl2dq['run_seconds'] = self.runparams['runtime']
cl2dq['fast'] = self.runparams['fast']
self.cl2dqdata=cl2dq
### finish alignment run
alignrunq['cl2drun'] = cl2dq
alignrunq['hidden'] = False
alignrunq['runname'] = self.runparams['runname']
alignrunq['description'] = self.runparams['description']
alignrunq['lp_filt'] = self.runparams['lowpass']
alignrunq['hp_filt'] = self.runparams['highpass']
alignrunq['bin'] = self.runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
if self.runparams['align'] is True: ### option to create aligned stack
alignstackq['imagicfile'] = alignimagicfile
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
alignstackq['iteration'] = self.runparams['maxiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
if self.runparams['align'] is True: ### option to create aligned stack
alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: "+alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(self.runparams['stackid'])
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
alignstackq['description'] = self.runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.runparams['numpart']
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def createAlignedStacks(self, partlist):
stackid = self.params['stackid']
stackdata = apStack.getOnlyStackData(stackid)
origstackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
imagesdict = apImagicFile.readImagic(origstackfile)
i = 0
t0 = time.time()
apDisplay.printMsg("rotating and shifting particles at "+time.asctime())
alignstack = []
while i < len(partlist):
partimg = imagesdict['images'][i]
partdict = partlist[i]
partnum = i+1
#print partnum, partdict, partimg.shape
if partdict['partnum'] != partnum:
apDisplay.printError("particle shifting "+str(partnum)+" != "+str(partdict))
xyshift = (partdict['xshift'], partdict['yshift'])
alignpartimg = apImage.xmippTransform(partimg, rot=partdict['inplane'],
shift=xyshift, mirror=partdict['mirror'])
alignstack.append(alignpartimg)
i += 1
apDisplay.printMsg("rotate then shift %d particles in %s"%(i,apDisplay.timeString(time.time()-t0)))
alignstackarray = numpy.asarray(alignstack)
self.alignimagicfile = "alignstack.hed"
apImagicFile.writeImagic(alignstackarray, self.alignimagicfile)
def start(self):
#new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
#make sure that old stack is numbered
apEMAN.checkStackNumbering(oldstack)
alignedstack = os.path.join(self.params['rundir'], 'ali.img')
badstack = os.path.join(self.params['rundir'], 'bad.img')
apStack.checkForPreviousStack(alignedstack)
#run centering algorithm
apStack.centerParticles(oldstack, self.params['mask'], self.params['maxshift'])
self.params['keepfile'] = os.path.join(self.params['rundir'],'keepfile.txt')
apEMAN.writeStackParticlesToFile(alignedstack, self.params['keepfile'])
if not os.path.isfile(alignedstack, ):
apDisplay.printError("No stack was created")
#get number of particles
f = open(self.params['keepfile'], "r")
numparticles = len(f.readlines())
f.close()
self.params['description'] += (
(" ... %d eman centered substack id %d"
% (numparticles, self.params['stackid']))
)
apStack.commitSubStack(self.params, newname='ali.hed', centered=True)
apStack.averageStack(stack=alignedstack)
if (os.path.exists(badstack)):
apStack.averageStack(stack=badstack, outfile='badaverage.mrc')
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
#if self.params['description'] is None:
# apDisplay.printError("run description was not defined")
if self.params['numrefs'] is None:
apDisplay.printError("a number of classes was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if not self.params['fastmode'] in self.fastmodes:
apDisplay.printError("fast mode must be on of: "+str(self.fastmodes))
if not self.params['converge'] in self.convergemodes:
apDisplay.printError("converge mode must be on of: "+str(self.convergemodes))
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
def checksizes(self):
newnots = apFile.numImagesInStack(self.params['notstack'])
newtilts = apFile.numImagesInStack(self.params['tiltstack'])
stackdata1 = apStack.getOnlyStackData(self.params['stack1'], msg=False)
stackpath1 = os.path.join(stackdata1['path']['path'], stackdata1['name'])
oldnots = apFile.numImagesInStack(stackpath1)
stackdata2 = apStack.getOnlyStackData(self.params['stack2'], msg=False)
stackpath2 = os.path.join(stackdata2['path']['path'], stackdata2['name'])
oldtilts = apFile.numImagesInStack(stackpath2)
if newtilts != oldtilts:
apDisplay.printWarning("tilted stack are different sizes: %d vs. %d part"%(newtilts,oldtilts))
else:
apDisplay.printMsg("tilted stack are the sames sizes: %d part"%(newtilts))
if newnots != oldnots:
apDisplay.printWarning("untilted stack are different sizes: %d vs. %d part"%(newnots,oldnots))
else:
apDisplay.printMsg("untilted stack are the sames sizes: %d part"%(newnots))
def setRunDir(self): stackdata = apStack.getOnlyStackData(self.params['tiltstackid'], msg=False) path = stackdata['path']['path'] uppath = os.path.dirname(os.path.dirname(os.path.abspath(path))) classliststr = operations.intListToString(self.classlist) self.params['rundir'] = os.path.join(uppath, "rctvolume", self.params['runname'] )
def setRunDir(self):
stackdata = apStack.getOnlyStackData(self.params["partstackid"], msg=False)
path = stackdata["path"]["path"]
uppath = os.path.dirname(os.path.dirname(os.path.abspath(path)))
self.params["rundir"] = os.path.join(uppath, "SpiderRecon", self.params["runname"])
print self.params["rundir"]
def checkConflicts(self):
if self.params['tiltstackid'] is None:
apDisplay.printError("Enter a tilted stack ID")
if self.params['notstackid'] is None:
apDisplay.printError("Enter a untilted stack ID")
if self.params['runname'] is None:
apDisplay.printError("Enter run name")
self.tiltstackdata = apStack.getOnlyStackData(self.params['tiltstackid'])
def setRunDir(self): """ This funcion is only run when the user does not specifiy 'rundir' """ self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) path = self.stackdata['path']['path'] uppath = os.path.abspath(os.path.join(path, "../..")) self.params['rundir'] = os.path.join(uppath, "align", self.params['runname'])
def insertOtrRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq['type'] = "fsc" fscresq['half'] = self.fscresolution fscresq['fscfile'] = "fscdata"+self.timestamp+".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq['type'] = "rmeasure" rmeasureq['half'] = self.rmeasureresolution rmeasureq['fscfile'] = None ### insert rct run data otrrunq = appiondata.ApOtrRunData() otrrunq['runname'] = self.params['runname'] tempstr = "" for cnum in self.classlist: tempstr += str(cnum)+"," classliststr = tempstr[:-1] otrrunq['classnums'] = classliststr otrrunq['numiter'] = self.params['numiters'] otrrunq['euleriter'] = self.params['refineiters'] otrrunq['maskrad'] = self.params['radius'] otrrunq['lowpassvol'] = self.params['lowpassvol'] otrrunq['highpasspart'] = self.params['highpasspart'] otrrunq['median'] = self.params['median'] otrrunq['description'] = self.params['description'] otrrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) otrrunq['alignstack'] = self.alignstackdata otrrunq['tiltstack'] = apStack.getOnlyStackData(self.params['tiltstackid']) otrrunq['numpart'] = self.numpart otrrunq['fsc_resolution'] = fscresq otrrunq['rmeasure_resolution'] = rmeasureq if self.params['commit'] is True: otrrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['otrrun'] = otrrunq densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25) densq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] densq['boxsize'] = self.getBoxSize() densq['lowpass'] = self.params['lowpassvol'] densq['highpass'] = self.params['highpasspart'] densq['mask'] = self.params['radius'] #densq['iterid'] = self.params['numiters'] densq['description'] = self.params['description'] densq['resolution'] = self.fscresolution densq['rmeasure'] = self.rmeasureresolution densq['session'] = apStack.getSessionDataFromStackId(self.params['tiltstackid']) densq['md5sum'] = apFile.md5sumfile(volfile) if self.params['commit'] is True: densq.insert() return
def insertOtrRun(self, volfile):
### setup resolutions
fscresq = appiondata.ApResolutionData()
fscresq["type"] = "fsc"
fscresq["half"] = self.fscresolution
fscresq["fscfile"] = "fscdata" + self.timestamp + ".fsc"
rmeasureq = appiondata.ApResolutionData()
rmeasureq["type"] = "rmeasure"
rmeasureq["half"] = self.rmeasureresolution
rmeasureq["fscfile"] = None
### insert rct run data
otrrunq = appiondata.ApOtrRunData()
otrrunq["runname"] = self.params["runname"]
tempstr = ""
for cnum in self.classlist:
tempstr += str(cnum) + ","
classliststr = tempstr[:-1]
otrrunq["classnums"] = classliststr
otrrunq["numiter"] = self.params["numiters"]
otrrunq["euleriter"] = self.params["refineiters"]
otrrunq["maskrad"] = self.params["radius"]
otrrunq["lowpassvol"] = self.params["lowpassvol"]
otrrunq["highpasspart"] = self.params["highpasspart"]
otrrunq["median"] = self.params["median"]
otrrunq["description"] = self.params["description"]
otrrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
otrrunq["alignstack"] = self.alignstackdata
otrrunq["tiltstack"] = apStack.getOnlyStackData(self.params["tiltstackid"])
otrrunq["numpart"] = self.numpart
otrrunq["fsc_resolution"] = fscresq
otrrunq["rmeasure_resolution"] = rmeasureq
if self.params["commit"] is True:
otrrunq.insert()
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq["otrrun"] = otrrunq
densq["path"] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile)))
densq["name"] = os.path.basename(volfile)
densq["hidden"] = False
densq["norm"] = True
densq["symmetry"] = appiondata.ApSymmetryData.direct_query(25)
densq["pixelsize"] = apStack.getStackPixelSizeFromStackId(self.params["partstackid"]) * self.params["bin"]
densq["boxsize"] = self.getBoxSize()
densq["lowpass"] = self.params["lowpassvol"]
densq["highpass"] = self.params["highpasspart"]
densq["mask"] = self.params["radius"]
# densq['iterid'] = self.params['numiters']
densq["description"] = self.params["description"]
densq["resolution"] = self.fscresolution
densq["rmeasure"] = self.rmeasureresolution
densq["session"] = apStack.getSessionDataFromStackId(self.params["tiltstackid"])
densq["md5sum"] = apFile.md5sumfile(volfile)
if self.params["commit"] is True:
densq.insert()
return
def setRunDir(self): stackid = int(self.params['stackid']) #stackdata = apStack.getOnlyStackData(stackid, msg=False) stackdata = apStack.getOnlyStackData(stackid) print "stackdata: ", stackdata path = stackdata['path']['path'] #substitude the stacks with dbstacks folder uppath = os.path.dirname(os.path.abspath(path))[:-6]+"dbstacks" self.params['rundir'] = os.path.join(uppath, self.params['runname'])
def start(self):
self.rootname = self.params['stackname'].split(".")[0]
self.params['outputstack'] = os.path.join(self.params['rundir'], self.params['stackname'])
if os.path.isfile(self.params['outputstack']):
apFile.removeStack(self.params['outputstack'])
if self.params['eotest'] is True:
self.params['evenstack'] = os.path.splitext(self.params['outputstack'])[0]+'.even.hed'
if os.path.isfile(self.params['evenstack']):
apFile.removeStack(self.params['evenstack'])
self.params['oddstack'] = os.path.splitext(self.params['outputstack'])[0]+'.odd.hed'
if os.path.isfile(self.params['oddstack']):
apFile.removeStack(self.params['oddstack'])
classes = self.getClassData(self.params['reconid'], self.params['iter'])
stackid = apStack.getStackIdFromRecon(self.params['reconid'])
stackdata = apStack.getOnlyStackData(stackid)
stackpath = os.path.join(stackdata['path']['path'], stackdata['name'])
classkeys = classes.keys()
classkeys.sort()
classnum=0
keeplist = self.procKeepList()
finallist = []
apDisplay.printMsg("Processing "+str(len(classes))+" classes")
#loop through classes
for key in classkeys:
classnum+=1
if classnum%10 == 1:
sys.stderr.write("\b\b\b\b\b\b\b\b\b\b\b\b\b\b")
sys.stderr.write(str(classnum)+" of "+(str(len(classkeys))))
# loop through particles in class
classfile = self.rootname+"-class.lst"
classf = open(classfile, 'w')
classf.write('#LST\n')
nptcls=0
for ptcl in classes[key]['particles']:
# translate DB into EMAN
partnum = ptcl['particle']['particleNumber'] - 1
if partnum in keeplist:
if ptcl['mirror']:
mirror=1
else:
mirror=0
rot = ptcl['euler3']*math.pi/180.0
classf.write(
"%d\t%s\t%f,\t%f,%f,%f,%d\n" %
(partnum, stackpath, ptcl['quality_factor'],
rot, ptcl['shiftx'], ptcl['shifty'], mirror))
nptcls+=1
finallist.append(partnum)
classf.close()
if nptcls<1:
continue
self.makeClassAverages(classfile, self.params['outputstack'], classes[key], self.params['mask'])
if self.params['eotest'] is True:
self.makeEvenOddClasses(classfile, self.params['outputstack'], classes[key], self.params['mask'])
apFile.removeFile(classfile)
sys.stderr.write("\n")
finalfilename = self.rootname+"-keep.lst"
finalf = open(finalfilename, 'w')
finallist.sort()
for partnum in finallist:
finalf.write('%d\n' % (partnum,) )
finalf.close()
stackstr = str(stackdata.dbid)
reconstr = str(self.params['reconid'])
### recon 3d volumes
threedname = os.path.join(self.params['rundir'], self.rootname+"."+str(self.params['iter'])+"a.mrc")
emancmd = ( "make3d "+self.params['outputstack']+" out="
+threedname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" )
#print emancmd
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eman.log")
threednameb = os.path.join(self.params['rundir'], self.rootname+"."+str(self.params['iter'])+"b.mrc")
emancmd = ( "proc3d "+threedname+" "+threednameb
+" apix=1.63 norm=0,1 lp=8 origin=0,0,0 mask="+str(self.params['mask'])+"; echo '' " )
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eman.log")
if self.params['eotest'] is True:
# even
evenname = os.path.join(self.params['rundir'], self.rootname+"-even."+str(self.params['iter'])+"a.mrc")
if os.path.isfile(self.params['evenstack']):
evenemancmd = ( "make3d "+self.params['evenstack']+" out="
+evenname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" )
#print evenemancmd
apEMAN.executeEmanCmd(evenemancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eveneman.log")
else:
apDisplay.printWarning("file "+self.params['evenstack']+" does not exist")
# odd
oddname = os.path.join(self.params['rundir'], self.rootname+"-odd."+str(self.params['iter'])+"a.mrc")
if os.path.isfile(self.params['oddstack']):
oddemancmd = ( "make3d "+self.params['oddstack']+" out="
+oddname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" )
#print oddemancmd
apEMAN.executeEmanCmd(oddemancmd, verbose=False, showcmd=True, logfile=self.rootname+"-oddeman.log")
else:
apDisplay.printWarning("file "+self.params['oddstack']+" does not exist")
#eotest
fscout = os.path.join(self.params['rundir'], self.rootname+"-fsc.eotest")
if os.path.isfile(oddname) and os.path.isfile(evenname):
eotestcmd = "proc3d "+oddname+" "+evenname+" fsc="+fscout
apEMAN.executeEmanCmd(eotestcmd, verbose=True, showcmd=True)
else:
apDisplay.printWarning("could not perform eotest")
if os.path.isfile(fscout):
res = apRecon.getResolutionFromFSCFile(fscout, 160.0, 1.63)
apDisplay.printColor( ("resolution: %.5f" % (res)), "cyan")
resfile = self.rootname+"-res.txt"
f = open(resfile, 'a')
f.write("[ %s ]\nresolution: %.5f\n" % (time.asctime(), res))
f.close()
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### test insert to make sure data is not overwritten
self.params['runtime'] = 0
#self.checkDuplicateRefBasedRun()
### set box size
self.boxsize = int(
math.floor(self.stack['boxsize'] / self.params['bin'] / 2.0)) * 2
### convert stack to spider
spiderstack = self.createSpiderFile()
### create template stack
templatestack = self.createTemplateStack()
### run the alignment
aligntime = time.time()
usestack = spiderstack
oldpartlist = None
for i in range(self.params['numiter']):
iternum = i + 1
apDisplay.printColor("\n\nITERATION " + str(iternum), "green")
alignedstack, partlist = alignment.refBasedAlignParticles(
usestack,
templatestack,
spiderstack,
self.params['xysearch'],
self.params['xystep'],
self.params['numpart'],
self.params['numtemplate'],
self.params['firstring'],
self.params['lastring'],
iternum=iternum,
oldpartlist=oldpartlist)
oldpartlist = partlist
usestack = alignedstack
templatestack = self.updateTemplateStack(alignedstack, partlist,
iternum)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: " + spiderstack)
apFile.removeFile(spiderstack, warn=True)
### convert aligned stack to imagic
finalspistack = "aligned.spi"
shutil.move(alignedstack, finalspistack)
imagicstack = "aligned.hed"
apFile.removeStack(imagicstack)
emancmd = "proc2d " + finalspistack + " " + imagicstack
apEMAN.executeEmanCmd(emancmd, verbose=True)
### average stack
apStack.averageStack(imagicstack)
### calculate resolution for each reference
apix = self.stack['apix'] * self.params['bin']
self.calcResolution(partlist, imagicstack, apix)
if self.params['commit'] is True:
apDisplay.printMsg("committing results to DB")
self.params['runtime'] = aligntime
self.insertRefBasedRun(partlist, imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
### remove temporary files
apFile.removeFilePattern("alignments/alignedstack*.spi")
apFile.removeFile(finalspistack)
def start(self):
if self.params['rundir'] is None or not os.path.isdir(
self.params['rundir']):
apDisplay.printError("upload directory does not exist: " +
str(self.params['rundir']))
### create temp directory for extracting data
self.params['tmpdir'] = os.path.join(self.params['rundir'], "temp")
apParam.createDirectory(self.params['tmpdir'], warning=True)
### make sure that the stack & model IDs exist in database
emanJobFile = self.findEmanJobFile()
self.params['stack'] = apStack.getOnlyStackData(self.params['stackid'])
self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
self.params['model'] = appiondata.ApInitialModelData.direct_query(
self.params['modelid'])
self.params['boxsize'] = apStack.getStackBoxsize(
self.params['stackid'])
### parse out the refinement parameters from the log file
self.parseLogFile()
### parse out the message passing subclassification parameters from the job/log file
if self.params['package'] == 'EMAN/MsgP':
self.parseMsgPassingParams()
### convert class average files from old to new format
self.convertClassAvgFiles()
### get a list of the files in the directory
self.listFiles()
### create a refinementRun entry in the database
self.insertRefinementRun()
if self.params['euleronly'] is False:
### insert the Iteration info
for iteration in self.iterationdatas:
### if only uploading one iteration, skip to that one
if self.params['oneiter'] and int(
iteration['num']) != self.params['oneiter']:
continue
### if beginning at later iteration, skip to that one
if self.params['startiter'] and int(
iteration['num']) < self.params['startiter']:
continue
### if beginning at later iteration, skip to that one
if self.params['enditer'] and int(
iteration['num']) > self.params['enditer']:
continue
apDisplay.printColor(
"\nUploading iteration " + str(iteration['num']) + " of " +
str(len(self.iterationdatas)) + "\n", "green")
for i in range(75):
sys.stderr.write("#")
sys.stderr.write("\n")
self.insertIteration(iteration)
### calculate euler jumps
if self.params['commit'] is True:
reconrunid = self.params['refineRun'].dbid
stackid = self.params['stack'].dbid
if self.params['oneiter'] is None and len(self.iterationdatas) > 1:
apDisplay.printMsg("calculating euler jumpers for recon=" +
str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(
reconrunid, stackid)
### coran keep plot
if self.params['package'] == 'EMAN/SpiCoran':
apCoranPlot.makeCoranKeepPlot(reconrunid)
apRecon.setGoodBadParticlesFromReconId(reconrunid)
def insertAlignmentRun(self, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + self.params['runname'] +
"' and path already exist in database")
### setup Reference preparation parameters, if given
MRAq = appiondata.ApMultiRefAlignRunData()
if self.params['refs'] is True:
MRAq['lowpass_refs'] = self.params['lowpass_refs']
MRAq['thresh_refs'] = self.params['thresh_refs']
MRAq['maskrad_refs'] = self.params['maskrad_refs']
### setup Multi Reference Alignment Run
MRAq['mirror'] = self.params['mirror']
MRAq['center'] = self.params['center']
MRAq['alignment_type'] = self.params['alignment_type']
if self.params['alignment_type'].lower() == "all":
MRAq['first_alignment'] = self.params['first_alignment']
if self.params['alignment_type'].lower() == "brute_force":
MRAq['num_orientations'] = self.params['num_orientations']
MRAq['max_shift_orig'] = self.params['max_shift_orig']
if self.params['center'] is True:
MRAq['max_shift_this'] = self.params['max_shift_this']
MRAq['samp_param'] = self.params['samp_param']
MRAq['min_radius'] = self.params['minrad']
MRAq['max_radius'] = self.params['minrad']
MRAq['numiter'] = self.params['numiter']
### finish alignment run
alignrunq['imagicMRA'] = MRAq
alignrunq['hidden'] = False
alignrunq['description'] = self.params['description']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['bin'] = self.params['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = "alignstack.hed"
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = "references.hed"
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: " + imagicfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: " +
refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackId'])
alignstackq['boxsize'] = self.params['boxsize']
alignstackq['pixelsize'] = self.params['apix']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def start(self):
### get stack data
notstackdata = apStack.getOnlyStackData(self.params['notstackid'])
tiltstackdata = apStack.getOnlyStackData(self.params['tiltstackid'])
### get good particle numbers
includeParticle, tiltParticlesData = self.getGoodAlignParticles()
self.numpart = len(includeParticle)
### make doc file of Euler angles
eulerfile = self.makeEulerDoc(tiltParticlesData)
### write kept particles to file
self.params['keepfile'] = os.path.join(
self.params['rundir'], "keepfile" + self.timestamp + ".lst")
apDisplay.printMsg("writing to keepfile " + self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
for partnum in includeParticle:
kf.write(str(partnum) + "\n")
kf.close()
### make new stack of tilted particle from that run
tiltstackfile = os.path.join(tiltstackdata['path']['path'],
tiltstackdata['name'])
rctstackfile = os.path.join(self.params['rundir'],
"rctstack" + self.timestamp + ".hed")
apFile.removeStack(rctstackfile, warn=False)
apStack.makeNewStack(tiltstackfile, rctstackfile,
self.params['keepfile'])
spiderstack = self.convertStackToSpider(rctstackfile)
#self.mirrorParticles(tiltParticlesData, spiderstack)
### iterations over volume creation
### back project particles into filter volume
volfile = os.path.join(self.params['rundir'],
"volume%s-%03d.spi" % (self.timestamp, 0))
backproject.backprojectCG(spiderstack,
eulerfile,
volfile,
numpart=self.numpart,
pixrad=self.params['radius'])
alignstack = spiderstack
### center/convert the volume file
mrcvolfile = self.processVolume(volfile, 0)
for i in range(self.params['numiters']):
looptime = time.time()
iternum = i + 1
apDisplay.printMsg("running backprojection iteration " +
str(iternum))
### xy-shift particles to volume projections
alignstack = backproject.rctParticleShift(
volfile,
alignstack,
eulerfile,
iternum,
numpart=self.numpart,
pixrad=self.params['radius'],
timestamp=self.timestamp)
apFile.removeFile(volfile)
### back project particles into better volume
volfile = os.path.join(
self.params['rundir'],
"volume%s-%03d.spi" % (self.timestamp, iternum))
backproject.backproject3F(alignstack,
eulerfile,
volfile,
numpart=self.numpart)
### center/convert the volume file
mrcvolfile = self.processVolume(volfile, iternum)
apDisplay.printColor(
"finished volume refinement loop in " +
apDisplay.timeString(time.time() - looptime), "cyan")
### optimize Euler angles
#NOT IMPLEMENTED YET
### perform eotest
if self.params['eotest'] is True:
self.runEoTest(alignstack, eulerfile)
self.runRmeasure()
### insert volumes into DB
self.insertRctRun(mrcvolfile)
def start(self):
### get stack parameteres
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackId'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackId'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackId'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### copy stack into working directory
if os.path.isfile(self.stack['file']):
apDisplay.printColor("copying stack into running directoroy",
"cyan")
if self.stack['file'][-4:] == ".img" or self.stack['file'][
-4:] == ".hed":
strippedfile = self.stack['file'][:-4]
else:
strippedfile = self.stack['file']
while os.path.isfile(
os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(
os.path.join(self.params['rundir'], "start.img"))
emancmd = "proc2d "+strippedfile+".hed "+os.path.join(self.params['rundir'], "start.hed ")+\
"first=0 last="+str(self.params['numpart']-1)
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("stack not found in database")
### get template stack parameters
self.templatestack = {}
self.templatestack[
'data'] = appiondata.ApTemplateStackData.direct_query(
self.params['templateStackId'])
self.templatestack['apix'] = self.templatestack['data']['apix']
self.templatestack['boxsize'] = self.templatestack['data']['boxsize']
self.templatestack['file'] = os.path.join(
self.templatestack['data']['path']['path'],
self.templatestack['data']['templatename'])
self.templatestack['numimages'] = self.templatestack['data'][
'numimages']
### copy templates into working directory
if os.path.isfile(self.templatestack['file']):
apDisplay.printColor("copying templates into running directoroy",
"cyan")
ts = os.path.join(self.params['rundir'], "references.img")
while os.path.isfile(ts):
apFile.removeStack(ts)
if self.templatestack['file'][-4:] == ".img" or self.templatestack[
'file'][-4:] == ".hed":
strippedfile = self.templatestack['file'][:-4]
else:
strippedfile = self.templatestack['file']
emancmd = "proc2d " + strippedfile + ".img " + ts
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("template stack not found in database")
### set new pixelsize
if self.params['bin'] is not None and self.params['bin'] != 0:
self.params['apix'] = float(self.stack['apix']) * int(
self.params['bin'])
else:
self.params['apix'] = self.stack['apix']
### scale, low-pass, and high-pass filter stack ... do this with imagic, because it determines the appropriate boxsizes
scalingbatchfile = self.createImagicBatchFileScaling()
preptime = time.time()
proc = subprocess.Popen("chmod 775 " + str(scalingbatchfile),
shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(scalingbatchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(
os.path.join(self.params['rundir'], "prepareStack.log"))
apDisplay.printColor(
"finished IMAGIC in " +
apDisplay.timeString(time.time() - preptime), "cyan")
### set new boxsize, done only after scaling is complete
if self.params['bin'] is not None:
self.params['boxsize'] = apFile.getBoxSize(
os.path.join(self.params['rundir'], "start.hed"))[0]
else:
self.params['boxsize'] = self.stack['boxsize']
### make sure template stack boxsize matches that of the input stack
if self.params['apix'] != self.templatestack['apix'] or self.params[
'boxsize'] != self.templatestack['boxsize']:
self.scaleTemplates()
starttime = time.time()
print self.params
print "... stack pixel size: " + str(self.params['apix'])
print "... stack box size: " + str(self.params['boxsize'])
apDisplay.printColor(
"Running IMAGIC .batch file: See multiReferenceAlignment.log file for details",
"cyan")
### create IMAGIC batch file
batchfile = self.createImagicBatchFileMRA()
### execute IMAGIC batch file
aligntime0 = time.time()
proc = subprocess.Popen("chmod 775 " + str(batchfile), shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(batchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(
os.path.join(self.params['rundir'], "multiReferenceAlignment.log"))
apDisplay.printColor(
"finished IMAGIC in " +
apDisplay.timeString(time.time() - aligntime0), "cyan")
### get particle parameters (shift, rotate, refnum, mirror, ccc)
partparams = self.getParticleParams()
### average stack
alignstack = os.path.join(self.params['rundir'], "alignstack.hed")
apStack.averageStack(alignstack)
### normalize particles (otherwise found problems in viewing with stackviewer)
emancmd = "proc2d " + alignstack + " " + alignstack + ".norm.hed norm"
while os.path.isfile(alignstack + ".norm.img"):
apFile.removeStack(alignstack + ".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(alignstack + ".norm.hed", alignstack)
os.rename(alignstack + ".norm.img", alignstack[:-4] + ".img")
### normalize references
emancmd = "proc2d " + ts + " " + ts + ".norm.hed norm"
while os.path.isfile(ts + ".norm.img"):
apFile.removeStack(ts + ".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(ts + ".norm.hed", ts)
os.rename(ts + ".norm.img", ts[:-4] + ".img")
### remove copied stack
while os.path.isfile(os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(os.path.join(self.params['rundir'],
"start.img"))
### insert run into database
self.insertAlignmentRun(insert=True)
self.insertParticlesIntoDatabase(partparams, insert=True)
def start(self):
#new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
apStackFormat.linkFormattedStack(stackdata, self.params['format'],
'test')
def createStackData(self):
apDisplay.printColor("Starting upload of stack", "blue")
pathq = appiondata.ApPathData()
pathq['path'] = self.params['rundir']
manq = appiondata.ApManualParamsData()
manq['diam'] = self.params['diameter']
manq['trace'] = False
selectq = appiondata.ApSelectionRunData()
selectq['name'] = 'fakestack_' + self.params['runname']
selectq['hidden'] = True
selectq['path'] = pathq
selectq['session'] = self.sessiondata
selectq['manparams'] = manq
stackq = appiondata.ApStackData()
stackq['name'] = "start.hed"
stackq['path'] = pathq
stackq['description'] = self.params['description']
stackq['hidden'] = False
stackq['pixelsize'] = self.params['apix'] * 1e-10
stackq['boxsize'] = self.boxsize
stackq['centered'] = False
stackparamq = appiondata.ApStackParamsData()
stackparamq['boxSize'] = self.boxsize
stackparamq['bin'] = 1
stackparamq['phaseFlipped'] = self.params['ctfcorrect']
if self.params['ctfcorrect'] is True:
stackparamq['fileType'] = "manual"
stackparamq['fileType'] = "imagic"
stackparamq['normalized'] = self.params['normalize']
stackparamq['lowpass'] = 0
stackparamq['highpass'] = 0
stackrunq = appiondata.ApStackRunData()
stackrunq['stackRunName'] = self.params['runname']
stackrunq['stackParams'] = stackparamq
stackrunq['selectionrun'] = selectq
stackrunq['session'] = self.sessiondata
runsinstackq = appiondata.ApRunsInStackData()
runsinstackq['stack'] = stackq
runsinstackq['stackRun'] = stackrunq
if self.params['commit'] is True:
runsinstackq.insert()
if 'syncstackid' in self.params.keys(
) and self.params['syncstackid']:
stackdata = runsinstackq['stack']
stack2data = apStack.getOnlyStackData(
self.params['syncstackid'])
syncq = appiondata.ApSyncStackData(
stack1=stackdata,
stack2=stack2data,
synctype=self.params['synctype'])
syncq.insert()
### for each particle
sys.stderr.write("Starting particle upload")
for i in range(self.numpart):
if i % 100 == 0:
sys.stderr.write(".")
partq = appiondata.ApParticleData()
partq['image'] = None #We have no image, see if this works???
partq['selectionrun'] = selectq
partq['xcoord'] = int(i % 1000)
partq['ycoord'] = int(i / 1000)
partq['diameter'] = self.params['diameter']
stackpartq = appiondata.ApStackParticleData()
stackpartq['particleNumber'] = i + 1
stackpartq['stack'] = stackq
stackpartq['stackRun'] = stackrunq
stackpartq['particle'] = partq
stackpartq['mean'] = 0.0
stackpartq['stdev'] = 1.0
if self.params['commit'] is True:
stackpartq.insert()
sys.stderr.write("\n")
return
def insertAlignStackRunIntoDatabase(self, alignimagicfile):
apDisplay.printMsg("Inserting CL2D Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.runparams['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
# if uniquerun:
# apDisplay.printError("Run name '"+self.runparams['runname']+"' and path already exist in database")
### setup cl2d run
cl2dq = appiondata.ApCL2DRunData()
cl2dq['runname'] = self.runparams['runname']
cl2dq['run_seconds'] = self.runparams['runtime']
cl2dq['fast'] = self.runparams['fast']
self.cl2dqdata = cl2dq
### finish alignment run
alignrunq['cl2drun'] = cl2dq
alignrunq['hidden'] = False
alignrunq['runname'] = self.runparams['runname']
alignrunq['description'] = self.runparams['description']
alignrunq['lp_filt'] = self.runparams['lowpass']
alignrunq['hp_filt'] = self.runparams['highpass']
alignrunq['bin'] = self.runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
if self.runparams['align'] is True: ### option to create aligned stack
alignstackq['imagicfile'] = alignimagicfile
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = "part" + self.params[
'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1)
alignstackq['iteration'] = self.runparams['maxiter']
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
if self.runparams['align'] is True: ### option to create aligned stack
alignimagicfilepath = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: " +
alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: " +
refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(
self.runparams['stackid'])
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(
self.runparams['stackid']) * self.runparams['bin']
alignstackq['description'] = self.runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.runparams['numpart']
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def start(self):
#old stack size
stacksize = apStack.getNumberStackParticlesFromId(
self.params['stackid'])
# if exclude or include lists are not defined...
if self.params['exclude'] is None and self.params['include'] is None:
# if first and last are specified, create a file
if self.params['first'] is not None and self.params[
'last'] is not None:
stp = str(self.params['first'])
enp = str(self.params['last'])
fname = 'sub' + str(
self.params['stackid']) + '_' + stp + '-' + enp + '.lst'
self.params['keepfile'] = os.path.join(self.params['rundir'],
fname)
apDisplay.printMsg("Creating keep list: " +
self.params['keepfile'])
f = open(self.params['keepfile'], 'w')
for i in range(self.params['first'], self.params['last'] + 1):
f.write('%d\n' % (int(i) - 1))
f.close()
# generate the random list by giving number and create the file
elif self.params['random'] is not None:
#numOfRandomParticles = str(self.params['random'])
#fname = 'random'+str(self.params['stackid'])+'_'+numOfRandomParticles+'.lst'
fname = "random%d_%d.lst" % (self.params['stackid'],
self.params['random'])
self.params['keepfile'] = os.path.join(self.params['rundir'],
fname)
apDisplay.printMsg("Creating keep list: " +
self.params['keepfile'])
# create a file
f = open(self.params['keepfile'], 'w')
# generate a random sequence by giving size
randomList = random.sample(xrange(self.params['last']),
self.params['random'])
randomList.sort()
for partnum in randomList:
f.write('%d\n' % partnum)
f.close()
# if splitting, create files containing the split values
elif self.params['split'] > 1:
for i in range(self.params['split']):
fname = 'sub' + str(
self.params['stackid']) + '.' + str(i + 1) + '.lst'
self.params['keepfile'] = os.path.join(
self.params['rundir'], fname)
apDisplay.printMsg("Creating keep list: " +
self.params['keepfile'])
f = open(self.params['keepfile'], 'w')
for p in range(stacksize):
if (p % self.params['split']) - i == 0:
f.write('%i\n' % p)
f.close()
# if exclude-from option is specified, convert particles to exclude
elif self.params['excludefile'] is True:
oldkf = open(self.params['keepfile'])
partlist = []
for line in oldkf:
particle = line.strip()
try:
particle = int(particle)
except:
continue
partlist.append(particle)
oldkf.close()
# create new list excluding the particles
apDisplay.printMsg("Converting keep file to exclude file")
newkeepfile = "tmpnewkeepfile.txt"
newkf = open(newkeepfile, 'w')
for p in range(stacksize):
if p not in partlist:
newkf.write("%i\n" % p)
newkf.close()
self.params['keepfile'] = os.path.abspath(newkeepfile)
# otherwise, just copy the file
elif not os.path.isfile(os.path.basename(self.params['keepfile'])):
shutil.copy(self.params['keepfile'],
os.path.basename(self.params['keepfile']))
# if either exclude or include lists is defined
elif self.params['exclude'] or self.params['include']:
### list of particles to be excluded
excludelist = []
if self.params['exclude'] is not None:
excludestrlist = self.params['exclude'].split(",")
for excld in excludestrlist:
excludelist.append(int(excld.strip()))
apDisplay.printMsg("Exclude list: " + str(excludelist))
### list of particles to be included
includelist = []
if self.params['include'] is not None:
includestrlist = self.params['include'].split(",")
for incld in includestrlist:
includelist.append(int(incld.strip()))
apDisplay.printMsg("Include list: " + str(includelist))
#new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
newname = stackdata['name']
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
#if include or exclude list is given...
if self.params['include'] is not None or self.params[
'exclude'] is not None:
includeParticle = []
excludeParticle = 0
for partnum in range(stacksize):
if includelist and partnum in includelist:
includeParticle.append(partnum)
elif excludelist and not partnum in excludelist:
includeParticle.append(partnum)
else:
excludeParticle += 1
includeParticle.sort()
### write kept particles to file
self.params['keepfile'] = os.path.join(
self.params['rundir'], "keepfile-" + self.timestamp + ".list")
apDisplay.printMsg("writing to keepfile " +
self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
for partnum in includeParticle:
kf.write(str(partnum) + "\n")
kf.close()
#get number of particles
numparticles = len(includeParticle)
if excludelist:
self.params['description'] += (
" ... %d particle substack of stackid %d" %
(numparticles, self.params['stackid']))
elif includelist:
self.params['description'] += (
" ... %d particle substack of stackid %d" %
(numparticles, self.params['stackid']))
ogdescr = self.params['description']
for i in range(self.params['split']):
### always do this, if not splitting split=1
sb = os.path.splitext(stackdata['name'])
if self.params['first'] is not None and self.params[
'last'] is not None:
newname = sb[0] + '.' + str(self.params['first']) + '-' + str(
self.params['last']) + sb[-1]
elif self.params['random'] is not None:
newname = "%s-random%d%s" % (sb[0], self.params['random'],
sb[-1])
elif self.params['split'] > 1:
fname = 'sub' + str(
self.params['stackid']) + '.' + str(i + 1) + '.lst'
self.params['keepfile'] = os.path.join(self.params['rundir'],
fname)
newname = sb[0] + '.' + str(i + 1) + 'of' + str(
self.params['split']) + sb[-1]
newstack = os.path.join(self.params['rundir'], newname)
apStack.checkForPreviousStack(newstack)
#get number of particles
f = open(self.params['keepfile'], "r")
numparticles = len(f.readlines())
f.close()
self.params['description'] = ogdescr
self.params['description'] += (
(" ... %d particle substack of stackid %d" %
(numparticles, self.params['stackid'])))
#if splitting, add to description
if self.params['split'] > 1:
self.params['description'] += (" (%i of %i)" %
(i + 1, self.params['split']))
#create the new sub stack
if not self.params['correctbeamtilt']:
apStack.makeNewStack(oldstack,
newstack,
self.params['keepfile'],
bad=True)
else:
apBeamTilt.makeCorrectionStack(self.params['stackid'],
oldstack, newstack)
if not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
apStack.commitSubStack(self.params, newname, sorted=False)
apStack.averageStack(stack=newstack)
newstackid = apStack.getStackIdFromPath(newstack)
if self.params['meanplot'] is True:
apDisplay.printMsg(
"creating Stack Mean Plot montage for stackid")
apStackMeanPlot.makeStackMeanPlot(newstackid)
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.clipsize = int(
math.floor(self.stack['boxsize'] / self.params['bin'] / 2.0) *
self.params['bin'] * 2.0)
#self.checkRunNamePath()
### convert stack to spider
spiderstack = self.createSpiderFile()
#create template stack
templatestack = self.createTemplateStack()
#create orientation reference
orientref = self.createOrientationReference()
###################################################################
aligntime = time.time()
### create batch file
batchfilepath = self.setupBatchFile(spiderstack, templatestack,
orientref)
### run the spider alignment
apDisplay.printColor(
"Running iterative ref-classification and free-alignment with spider",
"cyan")
self.runSpiderBatch(batchfilepath, spiderstack)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
###################################################################
### remove unaligned spider stack
apDisplay.printMsg("Removing un-aligned stack: " + spiderstack)
apFile.removeFile(spiderstack, warn=False)
### check to be sure files exist
avgfile = os.path.join(self.params['rundir'],
"alignstack.spi") #class averages
if not os.path.isfile(avgfile):
apDisplay.printError(
"Final stack of aligned particles does not exist.")
### convert stacks to imagic
self.convertStack2Imagic("alignstack.spi")
self.convertStack2Imagic("avg.spi")
### make alignment average in mrc format
emancmd = "proc2d avg.spi average.mrc average"
apEMAN.executeEmanCmd(emancmd)
inserttime = time.time()
if self.params['commit'] is True:
apDisplay.printWarning("committing results to DB")
self.runtime = aligntime
self.insertEdIterRun(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: " +
apDisplay.timeString(inserttime))
def start(self):
### check for existing run
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
selectrundata = selectrunq.query(readimages=False)
if selectrundata:
apDisplay.printError("Runname already exists")
if self.params['ddstack']:
self.other_ddstack_used = []
self.dd = apDDprocess.DDStackProcessing()
self.dd.setDDStackRun(self.params['ddstack'])
self.newddstackrun = self.dd.getDDStackRun(show_msg=True)
### stack data
stackdata = apStack.getOnlyStackData(self.params['stackid'])
### stack particles
stackparts = apStack.getStackParticlesFromId(self.params['stackid'],
msg=True)
stackparts.reverse()
### selection run for first particle
oldselectrun = stackparts[0]['particle']['selectionrun']
### set selection run
manualparamsq = appiondata.ApManualParamsData()
manualparamsq['diam'] = self.getDiamFromSelectionRun(oldselectrun)
manualparamsq['oldselectionrun'] = oldselectrun
manualparamsq['trace'] = False
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['hidden'] = False
selectrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
selectrunq['session'] = apStack.getSessionDataFromStackId(
self.params['stackid'])
selectrunq['manparams'] = manualparamsq
### insert particles
apDisplay.printMsg("Inserting particles into database")
count = 0
t0 = time.time()
startmem = mem.active()
numpart = len(stackparts)
for stackpart in stackparts:
count += 1
if count > 10 and count % 100 == 0:
perpart = (time.time() - t0) / float(count + 1)
apDisplay.printColor(
"part %d of %d :: %.1fM mem :: %s/part :: %s remain" %
(count, numpart, (mem.active() - startmem) / 1024.,
apDisplay.timeString(perpart),
apDisplay.timeString(perpart * (numpart - count))),
"blue")
oldpartdata = stackpart['particle']
newpartq = appiondata.ApParticleData(initializer=oldpartdata)
newpartq['selectionrun'] = selectrunq
if self.params['ddstack']:
newimagedata = self.getNewImageFromDDStack(
oldpartdata['image'])
if newimagedata is False:
# no pick transferred
continue
newpartq['image'] = newimagedata
if self.params['commit'] is True:
newpartq.insert()
apDisplay.printMsg("Completed in %s" %
(apDisplay.timeString(time.time() - t0)))
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
# symmetry info
if self.params['sym'] == 'Icos':
self.params['symtype'] = 'I'
self.params['symfold'] = None
else:
self.params['symtype'] = self.params['sym'][0]
self.params['symfold'] = int(self.params['sym'][1:])
# eman "d" symmetry is spider "ci"
if self.params['symtype'].upper() == "D":
self.params['symtype'] = "CI"
# create symmetry doc file
sydoc = "sym.spi"
refine.symmetryDoc(self.params['symtype'], self.params['symfold'],
sydoc)
# convert incr to array of increments
ang_inc = self.params['incr'].split(',')
self.params['numiter'] = len(ang_inc)
self.params['increments'] = []
for i in range(0, self.params['numiter']):
self.params['increments'].append(int(ang_inc[i]))
# convert stack to spider stack
spiderstack = os.path.join(self.params['rundir'], 'start.spi')
operations.stackToSpiderStack(self.stack['file'],
spiderstack,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
numpart=self.params['numpart'])
# create filtered stack
spiderstackfilt = spiderstack
if (self.params['lowpass'] + self.params['highpass']) > 0:
spiderstackfilt = os.path.join(self.params['rundir'],
'start_filt.spi')
operations.stackToSpiderStack(self.stack['file'],
spiderstackfilt,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
lp=self.params['lowpass'],
hp=self.params['highpass'],
numpart=self.params['numpart'])
# rescale initial model if necessary
outvol = os.path.join(self.params['rundir'], "vol000.spi")
apModel.rescaleModel(self.params['modelid'],
outvol,
self.stack['boxsize'],
self.stack['apix'],
spider=True)
self.params['itervol'] = outvol
for iter in range(1, self.params['numiter'] + 1):
# create projections for projection matching
apDisplay.printMsg("creating reference projections of volume: %s" %
self.params['itervol'])
projs, numprojs, ang, sel = refine.createProjections(
incr=self.params['increments'][iter - 1],
boxsz=self.stack['boxsize'],
symfold=self.params['symfold'],
invol=self.params['itervol'],
rad=self.params['rad'],
)
# run reference-based alignment
apDisplay.printMsg("running reference-based alignment (AP MQ)")
apmqfile = "apmq%03d.spi" % iter
outang = "angular%03d.spi" % iter
shf = "shifts%03d.spi" % iter
shiftedStack = "parts_shifted.spi"
apFile.removeFile(shf)
refine.spiderAPMQ(
projs=projs,
numprojs=numprojs,
tsearch=self.params['xysearch'],
tstep=self.params['xystep'],
firstRing=self.params['firstring'],
lastRing=self.params['lastring'],
stackfile=spiderstackfilt,
nump=self.params['numpart'],
ang=ang,
apmqfile=apmqfile,
outang=outang,
nproc=self.params['proc'],
)
# use cross-correlation to find the sub-pixel alignment
# of the particles,
# results will be saved in "peakfile.spi"
apFile.removeFile(shiftedStack)
# don't use MPI here - for some reason slower?
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
apmqlist = refine.readDocFile(apmqfile)
avgccrot = 0
apDisplay.printMsg("creating shifted stack")
for p in range(0, self.params['numpart']):
ref = int(float(apmqlist[p][2]))
ccrot = float(apmqlist[p][3])
inplane = float(apmqlist[p][4])
avgccrot += ccrot
# invert the sign - ref projs will be rotated
inplane *= -1
# get corresponding projection
if (ref <= 0):
# mirror projection if necessary
ref *= -1
refimg = spyder.fileFilter(projs) + "@" + str(ref)
refine.mirrorImg(refimg, "_3", inMySpi=mySpi)
img = "_3"
else:
img = spyder.fileFilter(projs) + "@" + str(ref)
refine.rotAndShiftImg(img, "_2", inplane, inMySpi=mySpi)
refine.maskImg("_2",
"_3",
self.params['rad'],
"D",
"E",
center=int((self.stack['boxsize'] / 2) + 1),
inMySpi=mySpi)
# pad ref image & stack image to twice the size
refine.padImg("_3",
"_2",
2 * self.stack['boxsize'],
"N",
1,
1,
0,
inMySpi=mySpi)
stackimg = spyder.fileFilter(spiderstack) + "@" + str(p + 1)
refine.padImg(stackimg,
"_1",
2 * self.stack['boxsize'],
"B",
1,
1,
inMySpi=mySpi)
# calculate cross-correlation
refine.getCC("_1", "_2", "_1", inMySpi=mySpi)
# crop the correllation image to allowable shift amount
shift = int(self.params['allowedShift'] *
self.stack['boxsize'])
dim = 2 * shift + 1
topleftx = self.stack['boxsize'] - shift + 1
refine.windowImg("_1",
"_2",
dim,
topleftx,
topleftx,
inMySpi=mySpi)
# find the sub-pixel location of cc peak
mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17", "_2",
"0")
# create new stack of shifted particles
shpos = spyder.fileFilter(shiftedStack) + "@" + str(p + 1)
mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN")
mySpi.toSpiderQuiet("GP x17", "_2",
str(shift + 1) + "," + str(shift + 1))
refine.copyImg(stackimg, shpos, inMySpi=mySpi)
mySpi.toSpiderQuiet("ELSE")
#mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16")
mySpi.toSpiderQuiet("SH F", stackimg, shpos, "-x15,-x16")
mySpi.toSpiderQuiet("ENDIF")
# save shifts to file
mySpi.toSpiderQuiet("SD " + str(p + 1) + ",x15,x16,x17",
spyder.fileFilter(shf))
mySpi.toSpiderQuiet("SD E", spyder.fileFilter(shf))
mySpi.close()
# create class average images
refine.createClassAverages(
shiftedStack,
projs,
apmqfile,
numprojs,
self.stack['boxsize'],
shifted=True,
)
# rename class averages & variacnes for iteration
cmd = "/bin/mv classes.hed classes.%d.hed;" % iter
cmd += "/bin/mv classes.img classes.%d.img;" % iter
cmd += "/bin/mv variances.hed variances.%d.hed;" % iter
cmd += "/bin/mv variances.img variances.%d.img;" % iter
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
# calculate the stddev for the apmq cc's for throwing out particles
avgccrot /= self.params['numpart']
stdccrot = 0
for p in range(0, self.params['numpart']):
stdccrot += abs(float(apmqlist[p][3]) - avgccrot)
stdccrot /= self.params['numpart']
cccutoff = avgccrot + (stdccrot * self.params['keepsig'])
apDisplay.printMsg("average cc: %f" % avgccrot)
apDisplay.printMsg("setting cutoff to: %f" % cccutoff)
# create new selection file that only has particles with good cc's
selectfile = "select%03d.spi" % iter
apFile.removeFile(selectfile)
mySpi = spyder.SpiderSession(nproc=self.params['proc'],
dataext=".spi",
logo=False,
log=False)
i = 1
for p in range(0, self.params['numpart']):
ccrot = float(apmqlist[p][3])
if ccrot >= cccutoff:
mySpi.toSpiderQuiet("x11=%d" % (p + 1))
mySpi.toSpiderQuiet("SD %d,x11" % i,
spyder.fileFilter(selectfile))
i += 1
mySpi.close()
# calculate the new 3d structure using centered projections
# and the corrected angles from the angular doc file
apDisplay.printMsg("creating 3d volume")
out_rawvol = "vol_raw%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
selectfile,
rad=self.params['rad'],
ang=outang,
out=out_rawvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['bpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'],
)
else:
refine.backProjection(shiftedStack,
selectfile,
ang=outang,
out=out_rawvol,
sym=sydoc,
nproc=self.params['proc'])
# create even & odd select files
apDisplay.printMsg("creating even/odd volumes")
oddfile = "selectodd%03d.spi" % iter
evenfile = "selecteven%03d.spi" % iter
refine.docSplit(selectfile, oddfile, evenfile)
# get the even & odd volumesa
oddvol = "vol1%03d.spi" % iter
evenvol = "vol2%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
oddfile,
rad=self.params['rad'],
ang=outang,
out=oddvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'])
refine.iterativeBackProjection(
shiftedStack,
evenfile,
rad=self.params['rad'],
ang=outang,
out=evenvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'])
else:
refine.backProjection(
shiftedStack,
oddfile,
ang=outang,
out=oddvol,
sym=sydoc,
nproc=self.params['proc'],
)
refine.backProjection(
shiftedStack,
evenfile,
ang=outang,
out=evenvol,
sym=sydoc,
nproc=self.params['proc'],
)
# calculate the FSC
apDisplay.printMsg("calculating FSC")
fscfile = "fsc%03d.spi" % iter
emanfsc = "fsc.eotest.%d" % iter
refine.calcFSC(oddvol, evenvol, fscfile)
# convert to eman-style fscfile
refine.spiderFSCtoEMAN(fscfile, emanfsc)
# calculate the resolution at 0.5 FSC & write to file
res = apRecon.calcRes(emanfsc, self.stack['boxsize'],
self.stack['apix'])
restxt = "resolution.txt"
if iter == 1 and os.path.isfile(restxt):
os.remove(restxt)
resfile = open(restxt, "a")
resfile.write("iter %d:\t%.3f\n" % (iter, res))
resfile.close()
# filter & normalize the volume to be used as a reference in the next round
outvol = "vol%03d.spi" % iter
emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % (
out_rawvol, outvol, self.stack['apix'], res,
self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# create mrc files of volumes
emancmd = "proc3d %s %s apix=%.3f mask=%d norm" % (
out_rawvol, "threed.%da.mrc" % iter, self.stack['apix'],
self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % (
out_rawvol, "threed.%da.lp.mrc" % iter, self.stack['apix'],
res, self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.param['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# set this model as start for next iteration, remove previous
os.remove(self.params['itervol'])
os.remove(out_rawvol)
self.params['itervol'] = outvol
# clean up directory
apDisplay.printMsg("cleaning up directory")
if os.path.isfile(oddvol):
os.remove(oddvol)
if os.path.isfile(evenvol):
os.remove(evenvol)
if os.path.isfile(ang):
os.remove(ang)
if os.path.isfile(sel):
os.remove(sel)
if os.path.isfile(projs):
os.remove(projs)
if os.path.isfile(oddfile):
os.remove(oddfile)
if os.path.isfile(evenfile):
os.remove(evenfile)
if os.path.isfile(emanfsc) and os.path.isfile(fscfile):
os.remove(fscfile)
if os.path.isfile(shiftedStack):
os.remove(shiftedStack)
os.remove(self.params['itervol'])
def uploadResults(self):
if self.params['commit'] is False:
return
# Get the new file order
fh = open("sort_junk.sel", 'r')
lines = fh.readlines()
i = 0
fileorder = {}
for line in lines:
args = line.split()
if (len(args) > 1):
match = re.match('[A-Za-z]+([0-9]+)\.[A-Za-z]+',
(args[0].split('/'))[-1])
if (match):
filenumber = int(match.groups()[0])
fileorder[i] = filenumber
i += 1
fh.close()
# Produce a new stack
oldstack = apStack.getOnlyStackData(self.params['stackid'], msg=False)
newstack = appiondata.ApStackData()
newstack['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
newstack['name'] = "sorted.hed"
if newstack.query(results=1):
apDisplay.printError(
"A stack with these parameters already exists")
# Fill in data and submit
newstack['oldstack'] = oldstack
newstack['hidden'] = False
newstack['substackname'] = self.params['runname']
newstack['description'] = self.params['description']
newstack['pixelsize'] = oldstack['pixelsize']
newstack['boxsize'] = oldstack['boxsize']
newstack['junksorted'] = True
newstack.insert()
# Insert stack images
apDisplay.printMsg("Inserting stack particles")
count = 0
total = len(fileorder.keys())
if total == 0:
apDisplay.printError(
"No particles can be inserted in the sorted stack")
for i in fileorder.keys():
count += 1
if count % 100 == 0:
sys.stderr.write(
"\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b")
sys.stderr.write(
str(count) + " of " + (str(total)) + " complete")
# Get particle from the old stack
oldparticle = apStack.getStackParticle(self.params['stackid'],
fileorder[i] + 1)
# Insert particle
newparticle = appiondata.ApStackParticleData()
newparticle['particleNumber'] = i + 1
newparticle['stack'] = newstack
newparticle['stackRun'] = oldparticle['stackRun']
newparticle['particle'] = oldparticle['particle']
newparticle['mean'] = oldparticle['mean']
newparticle['stdev'] = oldparticle['stdev']
newparticle.insert()
apDisplay.printMsg(
"\n" + str(total) +
" particles have been inserted into the sorted stack")
# Insert runs in stack
apDisplay.printMsg("Inserting Runs in Stack")
runsinstack = apStack.getRunsInStack(self.params['stackid'])
for run in runsinstack:
newrunsq = appiondata.ApRunsInStackData()
newrunsq['stack'] = newstack
newrunsq['stackRun'] = run['stackRun']
newrunsq.insert()
apDisplay.printMsg("finished")
return
def checkConflicts(self):
### first get all stack data
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### modify boxsize and pixelsize according to binning
self.boxsize = self.stack['boxsize']
self.clipsize = int(
math.floor(self.boxsize / float(self.params['bin'] * 2))) * 2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d" %
(self.params['clipsize'], self.clipsize))
self.clipsize = self.params['clipsize']
self.apix = self.stack['apix']
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
self.boxsize = clipsize
self.apix = self.apix * self.params['bin']
### basic error checking
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['ncls'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 1000000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles " +
str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(stackfile)))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > 5000:
apDisplay.printWarning(
"initial model calculation may not work with less than 5000 particles"
)
self.mpirun = self.checkMPI()
# if self.mpirun is None:
# apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
### SIMPLE defaults and error checking
if self.params['ring2'] is None:
self.params['ring2'] = (self.boxsize / 2) - 2
if self.params['ncls'] > 2000:
apDisplay.printError(
"number of classes should be less than 2000 for subsequent ORIGAMI run to work"
)
if self.params['ncls'] > self.params['numpart']:
self.params['ncls'] = self.params['numpart'] / self.params['minp']
if self.params['mask'] is None:
self.params['mask'] = (self.boxsize / 2) - 2
if self.params['mw'] is None:
apDisplay.printError(
"please specify the molecular weight (in kDa)")
def start(self):
### new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
newstack = os.path.join(self.params['rundir'], stackdata['name'])
apStack.checkForPreviousStack(newstack)
includelist = []
excludelist = []
### list of classes to be excluded
if self.params['dropclasslist'] is not None:
excludestrlist = self.params['dropclasslist'].split(",")
for excludeitem in excludestrlist:
excludelist.append(int(excludeitem.strip()))
apDisplay.printMsg("Exclude list: "+str(excludelist))
### list of classes to be included
if self.params['keepclasslist'] is not None:
includestrlist = self.params['keepclasslist'].split(",")
for includeitem in includestrlist:
includelist.append(int(includeitem.strip()))
### or read from keepfile
elif self.params['keepfile'] is not None:
keeplistfile = open(self.params['keepfile'])
for line in keeplistfile:
if self.params['excludefrom'] is True:
excludelist.append(int(line.strip()))
else:
includelist.append(int(line.strip()))
keeplistfile.close()
apDisplay.printMsg("Include list: "+str(includelist))
### get particles from align or cluster stack
apDisplay.printMsg("Querying database for particles")
q0 = time.time()
if self.params['alignid'] is not None:
# DIRECT SQL STUFF
sqlcmd = "SELECT " + \
"apd.partnum, " + \
"apd.xshift, apd.yshift, " + \
"apd.rotation, apd.mirror, " + \
"apd.spread, apd.correlation, " + \
"apd.score, apd.bad, " + \
"spd.particleNumber, " + \
"ard.refnum "+ \
"FROM ApAlignParticleData apd " + \
"LEFT JOIN ApStackParticleData spd ON " + \
"(apd.`REF|ApStackParticleData|stackpart` = spd.DEF_id) " + \
"LEFT JOIN ApAlignReferenceData ard ON" + \
"(apd.`REF|ApAlignReferenceData|ref` = ard.DEF_id) " + \
"WHERE `REF|ApAlignStackData|alignstack` = %i"%(self.params['alignid'])
# These are AlignParticles
particles = sinedon.directq.complexMysqlQuery('appiondata',sqlcmd)
elif self.params['clusterid'] is not None:
clusterpartq = appiondata.ApClusteringParticleData()
clusterpartq['clusterstack'] = self.clusterstackdata
# These are ClusteringParticles
particles = clusterpartq.query()
apDisplay.printMsg("Completed in %s\n"%(apDisplay.timeString(time.time()-q0)))
### write included particles to text file
includeParticle = []
excludeParticle = 0
badscore = 0
badshift = 0
badspread = 0
f = open("test.log", "w")
count = 0
t0 = time.time()
apDisplay.printMsg("Parsing particle information")
# find out if there is alignparticle info:
is_cluster_p = False
# alignparticle is a key of any particle in particles if the latter is
# a CluateringParticle
if 'alignparticle' in particles[0]:
is_cluster_p = True
for part in particles:
count += 1
if is_cluster_p:
# alignpart is an item of ClusteringParticle
alignpart = part['alignparticle']
try:
classnum = int(part['refnum'])-1
except:
apDisplay.printWarning("particle %d was not put into any class" % (part['partnum']))
emanstackpartnum = alignpart['stackpart']['particleNumber']-1
else:
# particle has info from AlignedParticle as results of direct query
alignpart = part
try:
classnum = int(alignpart['refnum'])-1
except:
apDisplay.printWarning("particle %d was not put into any class" % (part['partnum']))
classnum = None
emanstackpartnum = int(alignpart['particleNumber'])-1
### check shift
if self.params['maxshift'] is not None:
shift = math.hypot(alignpart['xshift'], alignpart['yshift'])
if shift > self.params['maxshift']:
excludeParticle += 1
if classnum is not None:
f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum))
else:
f.write("%d\t%d\texclude\n"%(count, emanstackpartnum))
badshift += 1
continue
if self.params['minscore'] is not None:
### check score
if ( alignpart['score'] is not None
and alignpart['score'] < self.params['minscore'] ):
excludeParticle += 1
if classnum is not None:
f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum))
else:
f.write("%d\t%d\texclude\n"%(count, emanstackpartnum))
badscore += 1
continue
### check spread
if ( alignpart['spread'] is not None
and alignpart['spread'] < self.params['minscore'] ):
excludeParticle += 1
if classnum is not None:
f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum))
else:
f.write("%d\t%d\texclude\n"%(count, emanstackpartnum))
badspread += 1
continue
if classnum is not None:
if includelist and (classnum in includelist):
includeParticle.append(emanstackpartnum)
f.write("%d\t%d\t%d\tinclude\n"%(count, emanstackpartnum, classnum))
elif excludelist and not (classnum in excludelist):
includeParticle.append(emanstackpartnum)
f.write("%d\t%d\t%d\tinclude\n"%(count, emanstackpartnum, classnum))
else:
excludeParticle += 1
f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum))
else:
excludeParticle += 1
f.write("%d\t%d\texclude\n"%(count, emanstackpartnum))
f.close()
includeParticle.sort()
if badshift > 0:
apDisplay.printMsg("%d paricles had a large shift"%(badshift))
if badscore > 0:
apDisplay.printMsg("%d paricles had a low score"%(badscore))
if badspread > 0:
apDisplay.printMsg("%d paricles had a low spread"%(badspread))
apDisplay.printMsg("Completed in %s\n"%(apDisplay.timeString(time.time()-t0)))
apDisplay.printMsg("Keeping "+str(len(includeParticle))+" and excluding "+str(excludeParticle)+" particles")
### write kept particles to file
self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list")
apDisplay.printMsg("writing to keepfile "+self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
for partnum in includeParticle:
kf.write(str(partnum)+"\n")
kf.close()
### get number of particles
numparticles = len(includeParticle)
if excludelist:
self.params['description'] += ( " ... %d particle substack with %s classes excluded"
% (numparticles, self.params['dropclasslist']))
elif includelist:
self.params['description'] += ( " ... %d particle substack with %s classes included"
% (numparticles, self.params['keepclasslist']))
outavg = os.path.join(self.params['rundir'],"average.mrc")
### create the new sub stack
# first check if virtual stack
if not os.path.isfile(oldstack):
vstackdata=apStack.getVirtualStackParticlesFromId(self.params['stackid'])
vparts = vstackdata['particles']
oldstack = vstackdata['filename']
# get subset of virtualstack
vpartlist = [int(vparts[p]['particleNumber'])-1 for p in includeParticle]
if self.params['writefile'] is True:
apStack.makeNewStack(oldstack, newstack, vpartlist, bad=self.params['savebad'])
apStack.averageStack(stack=oldstack,outfile=outavg,partlist=vpartlist)
else:
if self.params['writefile'] is True:
apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=self.params['savebad'])
apStack.averageStack(stack=oldstack,outfile=outavg,partlist=includeParticle)
if self.params['writefile'] is True and not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
if self.params['commit'] is True:
apStack.commitSubStack(self.params,included=includeParticle)
newstackid = apStack.getStackIdFromPath(newstack)
apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=4)
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local temp file
proccmd = "proc2d " + self.stack['file'] + " temp.hed apix=" + str(
self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### process stack to final file
self.params['localstack'] = os.path.join(self.params['rundir'],
self.timestamp + ".hed")
proccmd = "proc2d temp.hed " + self.params[
'localstack'] + " apix=" + str(
self.stack['apix'] * self.params['bin'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
apFile.removeStack("temp.hed")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### convert stack into single spider files
templateselfile = self.initializeTemplates()
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -iter " + str(self.params['maxiter']) + " -ref " +
templateselfile + " -o " +
os.path.join(self.params['rundir'], "part" + self.timestamp) +
" -psi_step " + str(self.params['psistep']))
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-18 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using " + str(nproc - 1) + " processors!",
"green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun + " -np " + str(
nproc - 1) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
def start(self):
### get stack data
stackdata = apStack.getOnlyStackData(self.params['partstackid'])
###############################
# #
# Andres's refinement steps #
# #
###############################
print "\n"
apDisplay.printMsg("##################################")
apDisplay.printMsg("Starting Andres' refinement steps")
apDisplay.printMsg("##################################")
print "\n"
if self.params['initvol'] is not None:
spidervol = self.convertVolToSpider(mrcvolfile=self.params['initvol'], apix=self.params['initvolapix'])
if self.params['modelid'] is not None:
spidervol = self.convertVolToSpider(modelid=self.params['modelid'])
partstack = stackdata['path']['path']+"/"+stackdata['name']
spiderstack = self.convertStackToSpider(partstack)
partsnum = apStack.getNumberStackParticlesFromId(self.params['partstackid'])
for j in range(self.params['refineiters']):
iternum = j+1
appiondata.ApPathData.direct_query(1)
apDisplay.printMsg("Starting projection-matching refinement/XMIPP iteration "+str(iternum))
boxsize = self.getBoxSize()
### projection-matching refinement/XMIPP
apshout, apshstack, apsheuler, projstack, numprojs = self.projMatchRefine(spidervol, spiderstack, boxsize, partsnum, self.params['radius'], iternum)
apDisplay.printMsg("Calculating weighted cross-correlation coefficients")
### calculation of weighted cross-correlation coefficients
apshout_weighted = self.cccAPSH(apshout, iternum)
apDisplay.printMsg("Creating select files based on weighted cross-correlation coefficients")
### create select files based on calculated weighted-cross-correlation
corrSelect = self.makecccAPSHselectFile(apshout_weighted, iternum, factor=0.1)
### create volume file names
apshVolfile = os.path.join(self.params['rundir'], "apshVolume-BPCG-%03d.spi"%(iternum))
apshVolfile2 = os.path.join(self.params['rundir'], "apshVolume-BPRP-%03d.spi"%(iternum))
### run BPRP on selected particles
backprojectPWL.backprojectCG(apshstack, apsheuler, apshVolfile, partsnum, self.params['radius'])
self.APSHbackProject(apshstack, apsheuler, apshVolfile2, partsnum)
### center volume
filename = os.path.splitext(apshVolfile)[0]
apshVolFileCentered = filename+"_centered.spi"
backprojectPWL.centerVolume(apshVolfile, apshVolFileCentered)
### generate class averages
backprojectPWL.createClassAverages(apshstack,projstack,apshout,numprojs,boxsize,outclass="classes",rotated=True,shifted=True,dataext=".spi")
print "check~~!!!"
sys.exit(1)
### calculate FSC
### generate odd and even select files for FSC calculation
corrSelectOdd, corrSelectEven = self.splitOddEven(cnum, corrSelect, iternum)
fscout = self.runEoTest(corrSelectOdd, corrSelectEven, cnum, apshstack, apsheuler, iternum)
self.runRmeasure(apshVolFileCentered)
### filter volume
backproject.butterworthFscLP(apshVolFileCentered, fscout)
### reset file names for next round
volfile = apshVolFileCentered
eulerfile = apsheuler
mrcvolfile = self.processVolume(volfile, cnum, iternum)
print "\n"
apDisplay.printMsg("###########################")
apDisplay.printMsg("Done with iteration "+str(j+1)+"")
apDisplay.printMsg("###########################")
print "\n"
#if len(self.classlist) > 1:
#get a list of all unique combinations of volumes
# pairlist = self.computeClassVolPair()
### insert volumes into DB
self.insertOtrRun(mrcvolfile)
def checkConflicts(self):
### unpickle results or parse logfile, set default parameters if missing
os.chdir(os.path.abspath(self.params['rundir']))
if os.path.isdir(os.path.join(self.params['rundir'], "recon")):
apDisplay.printWarning(
'recon dir already exist, no need to unpack')
else:
self.unpackResults()
''' These are required error checks, everything else should be possible to obtain from the timestamped pickle file '''
stackdata = apStack.getOnlyStackData(self.params['stackid'])
self.runparams = {}
self.runparams['original_apix'] = (
stackdata['pixelsize'] / 1e-10
) # convert pixelsize to angstroms per pixel.
self.runparams['original_boxsize'] = stackdata['boxsize']
# some functions in readRunParameters needs the above params
self.runparams.update(self.readRunParameters())
### parameters recovered from runparameter file(s)
if not self.runparams.has_key('stackid'):
if self.params['stackid'] is not None:
self.runparams['stackid'] = self.params['stackid']
else:
apDisplay.printError(
"stack id must be specified for proper database insertion")
if not self.runparams.has_key('modelid'):
if self.params['modelid'] is not None:
self.runparams['modelid'] = self.params['modelid']
else:
apDisplay.printError(
"model id must be specified for proper database insertion")
if not self.runparams.has_key('NumberOfReferences'):
if self.params['NumberOfReferences'] is not None:
self.runparams['NumberOfReferences'] = self.params[
'NumberOfReferences']
else:
apDisplay.printError(
"number of references produced during the refinement needs to be specified"
)
if not self.runparams.has_key('numiter'):
if self.params['numiter'] is not None:
self.runparams['numiter'] = self.params['numiter']
else:
apDisplay.printError(
"number of iterations run during the refinement needs to be specified"
)
if not self.runparams.has_key('boxsize'):
if self.params['boxsize'] is not None:
self.runparams['boxsize'] = self.params['boxsize']
else:
apDisplay.printError(
"boxsize of the map / particles submitted for refinement needs to be specified"
)
if not self.runparams.has_key('apix'):
if self.params['apix'] is not None:
self.runparams['apix'] = self.params['apix']
else:
apDisplay.printError(
"pixelsize of the map / particles submitted for refinement needs to be specified"
)
if not self.runparams.has_key('symmetry'):
if self.params['symid'] is not None:
self.runparams['symmetry'] = apSymmetry.getSymmetryDataFromID(
self.params['symid'])
else:
apDisplay.printError(
"symmetry ID must be specified, you can input --symid=25 for an asymmetric reconstruction"
)
# access multiModelRefinementRun this way in case it is not present
if 'multiModelRefinementRun' in vars(self):
if not self.runparams.has_key(
'NumberOfReferences'
) and self.multiModelRefinementRun is True:
if self.params['NumberOfReferences'] is not None:
self.runparams['NumberOfReferences'] = self.params[
'NumberOfReferences']
else:
apDisplay.printError(
"number of output models in refinement needs to be specified for multi-model run"
)
else:
if self.params['NumberOfReferences'] is not None:
self.runparams['NumberOfReferences'] = self.params[
'NumberOfReferences']
if self.runparams['NumberOfReferences'] > 1:
self.multiModelRefinementRun = True
else:
self.multiModelRefinementRun = False
else:
apDisplay.printError(
"number of output models (references) in refinement needs to be specified for multi-model run"
)
if not self.runparams.has_key('rundir'):
self.runparams['rundir'] = self.params['rundir']
if not self.runparams.has_key('reconstruction_working_dir'):
self.runparams['reconstruction_working_dir'] = str(
self.package) + "_results"
if not self.runparams.has_key('mask'):
self.runparams['mask'] = None
if not self.runparams.has_key('imask'):
self.runparams['imask'] = None
if not self.runparams.has_key('alignmentInnerRadius'):
self.runparams['alignmentInnerRadius'] = None
if not self.runparams.has_key('alignmentOuterRadius'):
self.runparams['alignmentOuterRadius'] = None
if not self.runparams.has_key('angularSamplingRate'):
self.runparams['angularSamplingRate'] = None
### parameters specified for upload
if self.params['jobid'] is not None:
self.params['jobinfo'] = appiondata.ApAppionJobData.direct_query(
self.params['jobid'])
else:
jobid = self.tryToGetJobID()
if jobid is not False:
self.params[
'jobinfo'] = appiondata.ApAppionJobData.direct_query(
self.params['jobid'])
else:
self.params['jobinfo'] = None
if self.params[
'timestamp'] is None and self.package == "external_package":
apDisplay.printError("a timestamp (or some identifier) must be specified with the files. For example, the 3D mrc file " \
"for iteration 1 should be named 'recon_YourIdentifier_it001_vol001.mrc, in which case the timestamp should be specified " \
"as --timestamp=YourIdentifier")
elif self.params['timestamp'] is None and self.params[
'jobid'] is None and self.package != "external_package":
self.params['timestamp'] = apParam.makeTimestamp()
elif self.params['timestamp'] is None and self.params[
'jobid'] is not None:
timestamp = self.getTimestamp()
if timestamp is None:
self.params['timestamp'] = apParam.makeTimestamp()
# apDisplay.printError("please specify the timestamp associated with the refinement parameters, e.g. --timestamp=08nov25c07")
### basic refinement variables
self.initializeRefinementUploadVariables()
return
def start(self):
self.params['output_fileformat'] = 'mrc'
newstackname = 'framealigned.hed'
stackdata = apStack.getStackParticlesFromId(self.params['stackid'])
stackrundata = apStack.getOnlyStackData(self.params['stackid'])
apix = stackrundata['pixelsize'] * 1e10
kev = stackdata[0]['particle']['image']['scope']['high tension'] / 1000
origstackpath = os.path.join(stackrundata['path']['path'],
stackrundata['name'])
boxsize = stackdata[0]['stackRun']['stackParams']['boxSize']
binning = stackdata[0]['stackRun']['stackParams']['bin']
#determine camera type
cameratype = stackdata[0]['particle']['image']['camera']['ccdcamera'][
'name']
if self.params['override_camera'] is not None:
cameratype = self.params['override_camera']
#create sorted boxfiles
imagedict = {}
masterlist = []
for particle in stackdata:
parentimage = particle['particle']['image']['filename']
if parentimage in imagedict.keys():
imagedict[parentimage].append(particle['particle'])
else:
imagedict[parentimage] = []
imagedict[parentimage].append(particle['particle'])
index = len(imagedict[parentimage]) - 1
masterlist.append({
'particle': particle,
'key': parentimage,
'index': index
})
#print masterlist
for key in imagedict:
particlelst = imagedict[key]
parentimage = key
framespath = particlelst[0]['image']['session']['frame path']
print cameratype
if 'Gatan' in cameratype:
#prepare frames
print framespath
#prepare frame directory
framespathname = os.path.join(self.params['rundir'],
parentimage + '.frames')
if os.path.exists(framespathname):
pass
else:
os.mkdir(framespathname)
print framespathname
mrcframestackname = parentimage + '.frames.mrc'
print mrcframestackname
nframes = particlelst[0]['image']['camera']['nframes']
print "Extracting frames for", mrcframestackname
for n in range(nframes):
a = mrc.read(os.path.join(framespath, mrcframestackname),
n)
numpil.write(a,
imfile=os.path.join(framespathname,
'RawImage_%d.tif' % (n)),
format='tiff')
elif 'DE' in cameratype:
framespathname = os.path.join(framespath,
parentimage + '.frames')
print os.getcwd()
print framespathname
#generate DE script call
if os.path.exists(framespathname):
print "found frames for", parentimage
nframes = particlelst[0]['image']['camera']['nframes']
boxname = parentimage + '.box'
boxpath = os.path.join(framespathname, boxname)
shiftdata = {'scale': 1, 'shiftx': 0, 'shifty': 0}
#flatfield references
brightrefpath = particlelst[0]['image']['bright']['session'][
'image path']
brightrefname = particlelst[0]['image']['bright']['filename']
brightnframes = particlelst[0]['image']['bright']['camera'][
'nframes']
darkrefpath = particlelst[0]['image']['dark']['session'][
'image path']
darkrefname = particlelst[0]['image']['dark']['filename']
darknframes = particlelst[0]['image']['dark']['camera'][
'nframes']
brightref = os.path.join(brightrefpath, brightrefname + '.mrc')
darkref = os.path.join(darkrefpath, darkrefname + '.mrc')
print brightref
print darkref
apBoxer.processParticleData(particle['particle']['image'],
boxsize, particlelst, shiftdata,
boxpath)
print framespathname
#set appion specific options
self.params['gainreference_filename'] = brightref
self.params['gainreference_framecount'] = brightnframes
self.params['darkreference_filename'] = darkref
self.params['darkreference_framecount'] = darknframes
self.params['input_framecount'] = nframes
self.params['boxes_fromfiles'] = 1
#self.params['run_verbosity']=3
self.params['output_invert'] = 0
#self.params['radiationdamage_apix=']=apix
self.params['radiationdamage_voltage'] = kev
#self.params['boxes_boxsize']=boxsize
outpath = os.path.join(self.params['rundir'], key)
if os.path.exists(outpath):
shutil.rmtree(outpath)
os.mkdir(outpath)
command = ['deProcessFrames.py']
keys = self.params.keys()
keys.sort()
for key in keys:
param = self.params[key]
#print key, param, type(param)
if param == None or param == '':
pass
else:
option = '--%s=%s' % (key, param)
command.append(option)
command.append(outpath)
command.append(framespathname)
print command
if self.params['dryrun'] is False:
subprocess.call(command)
#recreate particle stack
for n, particledict in enumerate(masterlist):
parentimage = particledict['key']
correctedpath = os.path.join(self.params['rundir'], parentimage)
print correctedpath
if os.path.exists(correctedpath):
correctedparticle = glob.glob(
os.path.join(correctedpath,
('%s.*.region_%03d.*' %
(parentimage, particledict['index']))))
print os.path.join(correctedpath,
('%s.*.region_%03d.*' %
(parentimage, particledict['index'])))
print correctedparticle
#sys.exit()
command = ['proc2d', correctedparticle[0], newstackname]
if self.params['output_rotation'] != 0:
command.append('rot=%d' % self.params['output_rotation'])
if self.params['show_DE_command'] is True:
print command
subprocess.call(command)
else:
print "did not find frames for ", parentimage
command = [
'proc2d', origstackpath, newstackname, ('first=%d' % n),
('last=%d' % n)
]
print command
if self.params['dryrun'] is False:
subprocess.call(command)
#upload stack
#make keep file
self.params['keepfile'] = 'keepfile.txt'
f = open(self.params['keepfile'], 'w')
for n in range(len(masterlist)):
f.write('%d\n' % (n))
f.close()
apStack.commitSubStack(self.params, newname=newstackname)
apStack.averageStack(stack=newstackname)
print "Done!!!!"
def checkConflicts(self):
apDisplay.printMsg(
"/bin/rm -f *.hed *.img *.doc *.vol *.mrc *.spi *.log *~ *.pickle *.hist *.proj *.xmp *.sel *.basis *.original; rm -fr volume*"
)
### check for missing and duplicate entries
#if self.params['alignid'] is None and self.params['clusterid'] is None:
# apDisplay.printError("Please provide either --cluster-id or --align-id")
if self.params['alignid'] is not None and self.params[
'clusterid'] is not None:
apDisplay.printError(
"Please provide only one of either --cluster-id or --align-id")
if self.params['modelstr'] is None and self.params['nvol'] is None:
apDisplay.printError(
"Please provide model numbers or number of volumes")
elif self.params['modelstr'] is not None:
modellist = self.params['modelstr'].split(",")
self.params['modelids'] = []
for modelid in modellist:
self.params['modelids'].append(int(modelid))
### get the stack ID from the other IDs
if self.params['alignid'] is not None:
self.alignstackdata = appiondata.ApAlignStackData.direct_query(
self.params['alignid'])
self.params['stackid'] = self.alignstackdata['stack'].dbid
elif self.params['clusterid'] is not None:
self.clusterstackdata = appiondata.ApClusteringStackData.direct_query(
self.params['clusterid'])
self.alignstackdata = self.clusterstackdata['clusterrun'][
'alignstack']
self.params['stackid'] = self.alignstackdata['stack'].dbid
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
#if self.params['description'] is None:
# apDisplay.printError("run description was not defined")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if not self.params['fastmode'] in self.fastmodes:
apDisplay.printError("fast mode must be on of: " +
str(self.fastmodes))
maxparticles = 150000
minparticles = 50
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
if self.params['numpart'] < minparticles:
apDisplay.printError("not enough particles requested, min: " +
str(minparticles) + " requested: " +
str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles " +
str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(stackfile)))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
### find number of processors
if self.params['nproc'] is None:
self.nproc = apParam.getNumProcessors()
else:
self.nproc = self.params['nproc']
self.mpirun = self.checkMPI()
def commitToDatabase(self):
"""
insert the results into the database
"""
### expected result for an alignment run:
### 1. aligned particle stack in IMAGIC
### 2. rotation, shift, and quality parameters for each particle
### 3. which particles belongs to which class
### 4. stack file with the class averages
alignedstack = os.path.join(self.params['rundir'], "ptcl.hed")
refstack = os.path.join(self.params['rundir'], "iter.final.hed")
averagemrc = os.path.join(self.params['rundir'], "average.mrc")
apStack.averageStack(alignedstack, averagemrc)
particlemapping = self.determineClassOwnership()
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError(
"Run name '" + runparams['runname'] +
"' and path already exisclassmappingt in database")
### setup eman refine2d run
emanrefinetwodq = appiondata.ApEMANRefine2dRunData()
emanrefinetwodq['runname'] = self.params['runname']
emanrefinetwodq['run_seconds'] = time.time() - self.t0
emanrefinetwodq['num_iters'] = self.params['numiter']
emanrefinetwodq['num_classes'] = self.params['numclasses']
### finish alignment run
alignrunq['refine2drun'] = emanrefinetwodq
alignrunq['hidden'] = False
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['bin'] = self.params['bin']
### setup alignment stackalignimagicfile
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = os.path.basename(alignedstack)
alignstackq['avgmrcfile'] = os.path.basename(averagemrc)
alignstackq['refstackfile'] = os.path.basename(refstack)
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: " +
alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printErrrefqor("could not find reference stack file: " +
refstackfile)
### continue setting values
alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid']) * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = apFile.numImagesInStack(
alignimagicfilepath)
### inserting particles and references
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for emanpartnum in range(self.params['numpart']):
partnum = emanpartnum + 1
if partnum % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refnum = particlemapping[emanpartnum]
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### TODO: create mrc file
#refq['mrcfile'] = refbase+".mrc"
#reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
#if not os.path.isfile(reffile):
# emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc"
# apEMAN.executeEmanCmd(emancmd, verbose=False)
#if not os.path.isfile(reffile):
# apDisplay.printError("could not find reference file: "+reffile)
### TODO: get resolution
#refq['ssnr_resolution'] = TODO
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partnum)
alignpartq['stackpart'] = stackpartdata
### TODO: get the alignment parameters
#alignpartq['xshift'] = partdict['xshift']
#alignpartq['yshift'] = partdict['yshift']
#alignpartq['rotation'] = partdict['inplane']
#alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
### TODO: get the score
#alignpartq['score'] = partdict['score']
### insert
if self.params['commit'] is True:
alignpartq.insert()
return
def setRunDir(self):
self.stackdata = apStack.getOnlyStackData(self.params['stackid'],
msg=False)
path = self.stackdata['path']['path']
uppath = os.path.dirname(os.path.abspath(path))
self.params['rundir'] = os.path.join(uppath, self.params['runname'])
def start(self):
commit = self.params['commit']
description = self.params['description']
processdir = self.params['fulltomodir']
runname = self.params['runname']
offsetz = self.params['offsetz']
subbin = self.params['bin']
invert = self.params['invert']
fulltomodata = apTomo.getFullTomoData(self.params['fulltomoId'])
#subvolume making
if (self.params['selexonId'] is not None
or self.params['stackId']) and fulltomodata is not None:
sessiondata = fulltomodata['session']
seriesname = fulltomodata['name'].rstrip('_full')
fullbin = fulltomodata['bin']
if not fullbin:
apDisplay.printWarning(
"no binning in full tomogram, something is wrong, use alignment bin for now"
)
fullbin = fulltomodata['aligner']['alignrun']['bin']
fulltomopath = os.path.join(
fulltomodata['reconrun']['path']['path'],
seriesname + "_full.rec")
fulltomoheader = mrc.readHeaderFromFile(fulltomopath)
fulltomoshape = fulltomoheader['shape']
if self.params['sizez'] > fulltomoshape[1] * fullbin:
self.params['sizez'] = fulltomoshape[1] * fullbin
subrunname = self.params['subrunname']
volumeindex = apTomo.getLastVolumeIndex(fulltomodata) + 1
dimension = {
'x': int(self.params['sizex']),
'y': int(self.params['sizey']),
'z': int(self.params['sizez'])
}
zprojimagedata = fulltomodata['zprojection']
apDisplay.printMsg("getting pixelsize")
pixelsize = apTomo.getTomoPixelSize(
zprojimagedata) * fullbin * subbin
gtransform = [1, 0, 0, 1, 0, 0]
if self.params['selexonId']:
particles = apParticle.getParticles(zprojimagedata,
self.params['selexonId'])
if self.params['stackId']:
particles, stackparticles = apStack.getImageParticles(
zprojimagedata, self.params['stackId'])
stackdata = apStack.getOnlyStackData(self.params['stackId'])
for p, particle in enumerate(particles):
print particle['xcoord'], particle['ycoord'], fullbin
center = apTomo.transformParticleCenter(
particle, fullbin, gtransform)
size = (dimension['x'] / fullbin, dimension['y'] / fullbin,
dimension['z'] / fullbin)
volumename = 'volume%d' % (volumeindex, )
volumedir = os.path.join(processdir, subrunname + '/',
volumename + '/')
apParam.createDirectory(volumedir)
apImod.trimVolume(processdir, subrunname, seriesname,
volumename, center, offsetz, size, True)
long_volumename = seriesname + '_' + volumename
subvolumepath = os.path.join(processdir, runname + "/",
volumename + "/",
long_volumename + ".rec")
if subbin > 1 or invert:
apTomo.modifyVolume(subvolumepath, subbin, invert)
if commit:
subtomorundata = apTomo.insertSubTomoRun(
sessiondata, self.params['selexonId'],
self.params['stackId'], subrunname, invert, subbin)
subtomodata = apTomo.insertSubTomogram(
fulltomodata, subtomorundata, particle, offsetz,
dimension, volumedir, long_volumename, volumeindex,
pixelsize, description)
apTomo.makeMovie(subvolumepath)
apTomo.makeProjection(subvolumepath)
volumeindex += 1
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.checkNoRefRun()
### convert stack to spider
spiderstack = self.createSpiderFile()
### create initialization template
if self.params['initmethod'] == 'allaverage':
templatefile = self.averageTemplate()
elif self.params['initmethod'] == 'selectrand':
templatefile = self.selectRandomParticles()
elif self.params['initmethod'] == 'randpart':
templatefile = self.pickRandomParticle()
elif self.params['initmethod'] == 'template':
templatefile = self.getTemplate()
else:
apDisplay.printError("unknown initialization method defined: " +
str(self.params['initmethod']) + " not in " +
str(self.initmethods))
apDisplay.printColor("Running spider this can take awhile", "cyan")
### run the alignment
aligntime = time.time()
pixrad = int(
round(self.params['partrad'] / self.stack['apix'] /
self.params['bin']))
alignedstack, self.partlist = alignment.refFreeAlignParticles(
spiderstack,
templatefile,
self.params['numpart'],
pixrad,
self.params['firstring'],
self.params['lastring'],
rundir=".")
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: " + spiderstack)
apFile.removeFile(spiderstack, warn=False)
### convert stack to imagic
imagicstack = self.convertSpiderStack(alignedstack)
apFile.removeFile(alignedstack)
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = aligntime
self.insertNoRefRun(imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: " +
apDisplay.timeString(inserttime))
def insertRctRun(self, volfile):
### setup resolutions
fscresq = appiondata.ApResolutionData()
fscresq['type'] = "fsc"
fscresq['half'] = self.fscresolution
fscresq['fscfile'] = "fscdata" + self.timestamp + ".fsc"
rmeasureq = appiondata.ApResolutionData()
rmeasureq['type'] = "rmeasure"
rmeasureq['half'] = self.rmeasureresolution
rmeasureq['fscfile'] = None
### insert rct run data
rctrunq = appiondata.ApRctRunData()
rctrunq['runname'] = self.params['runname']
classliststr = operations.intListToString(self.classlist)
rctrunq['classnums'] = classliststr
rctrunq['numiter'] = self.params['numiters']
rctrunq['maskrad'] = self.params['radius']
rctrunq['lowpassvol'] = self.params['lowpassvol']
rctrunq['highpasspart'] = self.params['highpasspart']
rctrunq['lowpasspart'] = self.params['lowpasspart']
rctrunq['median'] = self.params['median']
rctrunq['description'] = self.params['description']
rctrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
rctrunq['alignstack'] = self.alignstackdata
rctrunq['tiltstack'] = apStack.getOnlyStackData(
self.params['tiltstackid'])
rctrunq['numpart'] = self.numpart
rctrunq['fsc_resolution'] = fscresq
rctrunq['rmeasure_resolution'] = rmeasureq
if self.params['commit'] is True:
rctrunq.insert()
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq['rctrun'] = rctrunq
densq['path'] = appiondata.ApPathData(
path=os.path.dirname(os.path.abspath(volfile)))
densq['name'] = os.path.basename(volfile)
densq['hidden'] = False
densq['norm'] = True
densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25)
densq['pixelsize'] = apStack.getStackPixelSizeFromStackId(
self.params['tiltstackid']) * self.params['tiltbin']
densq['boxsize'] = self.getBoxSize()
densq['lowpass'] = self.params['lowpassvol']
densq['highpass'] = self.params['highpasspart']
densq['mask'] = self.params['radius']
#densq['iterid'] = self.params['numiters']
densq['description'] = self.params['description']
densq['resolution'] = self.fscresolution
densq['rmeasure'] = self.rmeasureresolution
densq['session'] = apStack.getSessionDataFromStackId(
self.params['tiltstackid'])
densq['md5sum'] = apFile.md5sumfile(volfile)
if self.params['commit'] is True:
densq.insert()
return
def uploadResults(self):
if self.params['commit'] is False:
return
# Produce new stacks
oldstack = apStack.getOnlyStackData(self.params['notstackid'],
msg=False)
notstack = appiondata.ApStackData()
notstack['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
notstack['name'] = self.notstackdata['name']
if notstack.query(results=1):
apDisplay.printError(
"A stack with these parameters already exists")
tiltstack = appiondata.ApStackData()
tiltstack['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
tiltstack['name'] = self.tiltstackdata['name']
if tiltstack.query(results=1):
apDisplay.printError(
"A stack with these parameters already exists")
# Fill in data and submit
notstack['oldstack'] = oldstack
notstack['hidden'] = False
notstack['substackname'] = self.params['runname']
notstack['description'] = self.params[
'description'] + " ... tilt stack sorted"
notstack['pixelsize'] = oldstack['pixelsize']
notstack.insert()
tiltstack['oldstack'] = oldstack
tiltstack['hidden'] = False
tiltstack['substackname'] = self.params['runname']
tiltstack['description'] = self.params[
'description'] + " ... tilt stack sorted"
tiltstack['pixelsize'] = oldstack['pixelsize']
tiltstack.insert()
# Insert stack images
apDisplay.printMsg("Inserting stack particles")
count = 0
for partdict in parttree:
count += 1
if count % 100 == 0:
sys.stderr.write(
"\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b")
sys.stderr.write(
str(count) + " of " + (str(total)) + " complete")
# Get particles from the old stacks
oldnotparticle = apStack.getStackParticle(
self.params['notstackid'], partdict['not'])
oldtiltparticle = apStack.getStackParticle(
self.params['tiltstackid'], partdict['tilt'])
# Insert particle
notparticle = appiondata.ApStackParticleData()
notparticle.update(oldnotparticle)
notparticle['particleNumber'] = count
notparticle['stack'] = notstack
notparticle.insert()
tiltparticle = appiondata.ApStackParticleData()
tiltparticle.update(oldtiltparticle)
tiltparticle['particleNumber'] = count
tiltparticle['stack'] = tiltstack
tiltparticle.insert()
apDisplay.printMsg(
"\n%d particles have been inserted into the tilt synchronized stacks"
% (count))
# Insert runs in stack
apDisplay.printMsg("Inserting Runs in Stack")
runsinstack = apStack.getRunsInStack(self.params['notstackid'])
for run in runsinstack:
newrunsq = appiondata.ApRunsInStackData()
newrunsq['stack'] = notstack
newrunsq['stackRun'] = run['stackRun']
newrunsq.insert()
runsinstack = apStack.getRunsInStack(self.params['tiltstackid'])
for run in runsinstack:
newrunsq = appiondata.ApRunsInStackData()
newrunsq['stack'] = tiltstack
newrunsq['stackRun'] = run['stackRun']
newrunsq.insert()
apDisplay.printMsg("finished")
return
def insertRunIntoDatabase(self, alignimagicfile, runparams):
apDisplay.printMsg("Inserting MaxLike Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = runparams['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup max like run
maxlikeq = appiondata.ApMaxLikeRunData()
maxlikeq['runname'] = runparams['runname']
maxlikeq['run_seconds'] = runparams['runtime']
#maxlikeq['mask_diam'] = 2.0*runparams['maskrad']
maxlikeq['fast'] = runparams['fast']
maxlikeq['fastmode'] = runparams['fastmode']
maxlikeq['mirror'] = runparams['mirror']
maxlikeq['student'] = bool(runparams['student'])
maxlikeq['init_method'] = "xmipp default"
maxlikeq['job'] = self.getMaxLikeJob(runparams)
### finish alignment run
alignrunq['maxlikerun'] = maxlikeq
alignrunq['hidden'] = False
alignrunq['runname'] = runparams['runname']
alignrunq['description'] = runparams['description']
alignrunq['lp_filt'] = runparams['lowpass']
alignrunq['hp_filt'] = runparams['highpass']
alignrunq['bin'] = runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = alignimagicfile
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = "part"+self.params['timestamp']+"_average.hed"
alignstackq['iteration'] = self.lastiter
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: "+alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(runparams['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(runparams['stackid'])*runparams['bin']
alignstackq['description'] = runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = runparams['numpart']
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def makeNewStacks(self, parttree):
### untilted stack
self.notstackdata = apStack.getOnlyStackData(self.params['notstackid'])
notstackfile = os.path.join(self.notstackdata['path']['path'],
self.notstackdata['name'])
### tilted stack
if not self.tiltstackdata:
self.tiltstackdata = apStack.getOnlyStackData(
self.params['tiltstackid'])
tiltstackfile = os.path.join(self.tiltstackdata['path']['path'],
self.tiltstackdata['name'])
### make doc file of Euler angles
#eulerfile = self.makeEulerDoc(parttree)
eulerfile = os.path.join(self.params['rundir'],
"eulersdoc" + self.timestamp + ".spi")
if os.path.isfile(eulerfile):
apFile.removeFile(eulerfile)
count = 0
notstacklist = []
tiltstacklist = []
sizelimit = 2048
notbox = apImagicFile.getBoxsize(notstackfile)
tiltbox = apImagicFile.getBoxsize(tiltstackfile)
tiltstacks = []
notstacks = []
t0 = time.time()
for partdict in parttree:
### print friendly message
if count % 100 == 0:
backs = "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b"
sys.stderr.write(backs + backs + backs + backs)
if count > sizelimit:
esttime = (len(parttree) / float(count) -
1.0) * (time.time() - t0)
sys.stderr.write(
str(count) + " particles of " + str(len(parttree)) +
", " + apDisplay.timeString(esttime) + " remaining")
else:
sys.stderr.write(
str(count) + " particles of " + str(len(parttree)))
### save stacks to file to save memory
if count % sizelimit == 0:
if count > 1:
apDisplay.printMsg("Writing stacks to file")
t1 = time.time()
tiltname = os.path.join(self.params['rundir'],
"tiltstack%d.hed" % (count))
apFile.removeStack(tiltname)
apImagicFile.writeImagic(tiltstacklist,
tiltname,
msg=False)
tiltstacks.append(tiltname)
apDisplay.printMsg("finished tilted stack in " +
apDisplay.timeString(time.time() - t1))
t1 = time.time()
notname = os.path.join(self.params['rundir'],
"notstack%d.hed" % (count))
apFile.removeStack(notname)
apImagicFile.writeImagic(notstacklist, notname, msg=False)
notstacks.append(notname)
apDisplay.printMsg("finished untilted stack in " +
apDisplay.timeString(time.time() - t1))
### reset stacks
apDisplay.printMsg("Reset stacks in memory")
notstacklist = []
tiltstacklist = []
### increment count
count += 1
### write to Euler doc
self.appendEulerDoc(eulerfile, partdict['tilt'], count)
### untilted stack
notpartarray = apImagicFile.readSingleParticleFromStack(
notstackfile, partdict['not'], notbox, False)
notstacklist.append(notpartarray)
### tilted stack
tiltpartarray = apImagicFile.readSingleParticleFromStack(
tiltstackfile, partdict['tilt'], tiltbox, False)
tiltstacklist.append(tiltpartarray)
### write remaining particles to stack
if len(notstacklist) > 0:
apDisplay.printMsg("Writing stacks to file")
t1 = time.time()
tiltname = os.path.join(self.params['rundir'],
"tiltstack%d.hed" % (count))
apFile.removeStack(tiltname)
apImagicFile.writeImagic(tiltstacklist, tiltname, msg=False)
tiltstacks.append(tiltname)
apDisplay.printMsg("finished tilted stack in " +
apDisplay.timeString(time.time() - t1))
t1 = time.time()
notname = os.path.join(self.params['rundir'],
"notstack%d.hed" % (count))
apFile.removeStack(notname)
apImagicFile.writeImagic(notstacklist, notname, msg=False)
notstacks.append(notname)
apDisplay.printMsg("finished untilted stack in " +
apDisplay.timeString(time.time() - t1))
### merge NOT stack
notname = os.path.join(self.params['rundir'], "notstack.hed")
apImagicFile.mergeStacks(notstacks, notname)
for stackname in notstacks:
apFile.removeStack(stackname, warn=False)
### merge TILT stack
tiltname = os.path.join(self.params['rundir'], "tiltstack.hed")
apImagicFile.mergeStacks(tiltstacks, tiltname)
for stackname in tiltstacks:
apFile.removeStack(stackname, warn=False)
### upload results
if self.params['commit'] is True:
self.uploadResults()
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apParam.runCmd(proccmd, "EMAN", verbose=True)
if self.params['numpart'] != apFile.numImagesInStack(
self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -codes " + str(self.params['numrefs']) + " -iter " +
str(self.params['maxiter']) + " -o " + os.path.join(
self.params['rundir'], "part" + self.params['timestamp']))
if self.params['fast']:
xmippopts += " -fast "
if self.params['correlation']:
xmippopts += " -useCorrelation "
if self.params['classical']:
xmippopts += " -classicalMultiref "
if self.params['align']:
xmippopts += " -alignImages "
### use multi-processor command
apDisplay.printColor(
"Using " + str(self.params['nproc']) + " processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True)
mpiruncmd = self.mpirun + " -np " + str(
self.params['nproc']) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd,
package="Xmipp",
verbose=True,
showcmd=True,
logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(self.params['runtime']))
### minor post-processing
self.createReferenceStack()
self.parseOutput()
self.clearIntermediateFiles()
# self.readyUploadFlag()
apParam.dumpParameters(
self.params, "cl2d-" + self.params['timestamp'] + "-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-" +
self.params['timestamp'] +
"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(
self.runparams['stackid']) * self.runparams['bin']
self.Nlevels = len(
glob.glob("part" + self.params['timestamp'] + "_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part" + self.params[
'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels - 1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part" + self.params['timestamp'] +
"_level_%02d_.hed" % level, classnum + 1,
partdict[classnum], len(partdict))
