def backproject3F(stackfile, eulerdocfile, volfile, numpart, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
### setup
starttime = time.time()
stackfile = spyder.fileFilter(stackfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(stackfile+dataext):
apDisplay.printError("stack file not found: "+stackfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(volfile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("BP 3F",
stackfile+"@*****", #stack file
"1-%d"%(numpart), #number of particles
eulerdocfile, #angle doc file
"*", #input symmetry file, '*' for skip
volfile, #filename for volume
)
mySpider.close()
apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext=".spi"):
"""
project 3D volumes using given Euler angles
"""
starttime = time.time()
volfile = spyder.fileFilter(volfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
projstackfile = spyder.fileFilter(projstackfile)
if not os.path.isfile(volfile+dataext):
apDisplay.printError("volume file not found: "+volfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(projstackfile)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("PJ 3Q",
volfile, #input vol file
str(pixrad), #pixel radius
"1-%d"%(numpart), #number of particles
eulerdocfile, #Euler DOC file
projstackfile+"@*****", #output projections
)
mySpider.close()
apDisplay.printColor("finished projections in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def backprojectCG(stackfile, eulerdocfile, volfile, numpart, pixrad, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
### setup
starttime = time.time()
stackfile = spyder.fileFilter(stackfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(stackfile+dataext):
apDisplay.printError("stack file not found: "+stackfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(volfile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("BP CG",
stackfile+"@*****", #stack file
"1-%d"%(numpart), #number of particles
str(pixrad), #particle radius
eulerdocfile, #angle doc file
"N", #has symmetry?, does not work
volfile, #filename for volume
"%.1e,%.1f" % (1.0e-5, 0.0), #error, chi^2 limits
"%d,%d" % (25,1), #iterations, 1st derivative mode
"2000", #lambda - higher=less sensitive to noise
)
mySpider.close()
apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['numrefs'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
self.mpirun = self.checkMPI()
if self.mpirun is None:
apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
if self.params['nproc'] < 2:
apDisplay.printError("Only the MPI version of CL2D is currently supported, must run with > 1 CPU")
def checkConflicts(self):
if self.params['lowpass'] > 0:
apDisplay.printWarning(
"lowpass filter value greater than zero; will ignore")
if self.params['highpass'] > 0:
apDisplay.printWarning(
"highpass filter value greater than zero; will ignore")
self.params['highpass'] = None
self.params['lowpass'] = None
#if self.params['kfactor'] is not None and self.params['numslices'] is not None:
# apDisplay.printError("only one of 'kfactor' or 'numslices' can be defined")
if self.params[
'numslices'] is not None and self.params['numslices'] >= 15:
apDisplay.printError(
"too many slices defined by numslices, should be more like 2-6"
)
if self.params['sizerange'] is not None and self.params[
'sizerange'] >= 1.95 * self.params['diam']:
apDisplay.printError(
"size range has be less than twice the diameter")
### get number of processors:
nproc = apParam.getNumProcessors()
if not self.params['nproc']:
self.params['nproc'] = nproc
elif nproc < self.params['nproc']:
apDisplay.printWarning(
"Limiting number of processors to the %i that are available" %
nproc)
self.params['nproc'] = nproc
return
def checkConflicts(self):
if self.params['thresh'] is None:
apDisplay.printError("threshold was not defined")
### Check if we have templates
if self.params['templateliststr'] is None:
apDisplay.printError(
"template list was not specified, e.g. --template-list=34,56,12"
)
### Parse template list
oldtemplateids = self.params['templateliststr'].split(',')
self.params['templateIds'] = []
for tid in oldtemplateids:
templateid = abs(int(tid))
self.params['templateIds'].append(templateid)
if self.params['templatemirrors'] is True:
self.params['templateIds'].append(-1 * templateid)
### Check if we have ranges
if self.params['rangeliststr'] is None:
apDisplay.printError(
"range not specified, please provide range in the order of templateIds"
)
### Check that numbers of ranges and templates are equal
rangestrlist = self.params['rangeliststr'].split('x')
if len(oldtemplateids) != len(rangestrlist):
apDisplay.printError(
"the number of templates and ranges do not match")
### Parse range list
i = 0
for rangestr in rangestrlist:
self.params['range' + str(i)] = rangestr
angs = rangestr.split(",")
if not len(angs) == 3:
apDisplay.printError("the range is not defined correctly")
self.params['startang' + str(i + 1)] = float(angs[0])
self.params['endang' + str(i + 1)] = float(angs[1])
self.params['incrang' + str(i + 1)] = float(angs[2])
self.params['mirror' + str(i + 1)] = False
if self.params['templatemirrors'] is True:
i += 1
self.params['startang' + str(i + 1)] = float(angs[0])
self.params['endang' + str(i + 1)] = float(angs[1])
self.params['incrang' + str(i + 1)] = float(angs[2])
self.params['mirror' + str(i + 1)] = True
i += 1
### get number of processors:
nproc = apParam.getNumProcessors()
if not self.params['nproc']:
self.params['nproc'] = nproc
elif nproc < self.params['nproc']:
apDisplay.printWarning(
"Limiting number of processors to the %i that are available" %
nproc)
self.params['nproc'] = nproc
return
def start(self):
self.runtime = 0
self.checkCoranRun()
### convert stack to spider
self.alignstackdata = self.getAlignedStack()
maskpixrad = self.params['maskrad']/self.alignstackdata['pixelsize']/self.params['bin']
boxpixdiam = int(math.ceil(maskpixrad)+1)*2
if boxpixdiam*self.params['bin'] > self.alignstackdata['boxsize']:
boxpixdiam = math.floor(self.alignstackdata['boxsize']/self.params['bin'])
clippixdiam = boxpixdiam*self.params['bin']
apDisplay.printMsg("Pixel mask radius="+str(maskpixrad))
oldalignedstack = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['imagicfile'])
alignedstackname = re.sub("\.", "_", self.alignstackdata['imagicfile'])+".spi"
alignedstack = os.path.join(self.params['rundir'], alignedstackname)
apFile.removeFile(alignedstack)
emancmd = ("proc2d %s %s spiderswap shrink=%d clip=%d,%d edgenorm"
%(oldalignedstack,alignedstack,self.params['bin'],clippixdiam,clippixdiam))
if self.params['numpart'] is not None:
emancmd += " last=%d"%(self.params['numpart']-1)
numpart = self.params['numpart']
else:
numpart = self.getNumAlignedParticles()
apEMAN.executeEmanCmd(emancmd, verbose=True)
### get number of processors
nproc = apParam.getNumProcessors()
esttime = classification.estimateTime(numpart, maskpixrad)
apDisplay.printColor("Running spider this can take awhile, estimated time: "+\
apDisplay.timeString(esttime),"cyan")
### do correspondence analysis
corantime = time.time()
self.contriblist = classification.correspondenceAnalysis( alignedstack,
boxsize=boxpixdiam, maskpixrad=maskpixrad,
numpart=numpart, numfactors=self.params['numfactors'], nproc=nproc)
corantime = time.time() - corantime
### make dendrogram
dendrotime = time.time()
classification.makeDendrogram(numfactors=min(3,self.params['numfactors']),nproc=nproc)
dendrotime = time.time() - dendrotime
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = corantime
self.insertCoranRun(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apFile.removeFile(alignedstack, warn=True)
apDisplay.printMsg("Correspondence Analysis time: "+apDisplay.timeString(corantime))
apDisplay.printMsg("Make Dendrogram time: "+apDisplay.timeString(dendrotime))
apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def checkConflicts(self):
apDisplay.printMsg("/bin/rm -f *.hed *.img *.doc *.vol *.mrc *.spi *.log *~ *.pickle *.hist *.proj *.xmp *.sel *.basis *.original; rm -fr volume*")
### check for missing and duplicate entries
#if self.params['alignid'] is None and self.params['clusterid'] is None:
# apDisplay.printError("Please provide either --cluster-id or --align-id")
if self.params['alignid'] is not None and self.params['clusterid'] is not None:
apDisplay.printError("Please provide only one of either --cluster-id or --align-id")
if self.params['modelstr'] is None and self.params['nvol'] is None:
apDisplay.printError("Please provide model numbers or number of volumes")
elif self.params['modelstr'] is not None:
modellist = self.params['modelstr'].split(",")
self.params['modelids'] = []
for modelid in modellist:
self.params['modelids'].append(int(modelid))
### get the stack ID from the other IDs
if self.params['alignid'] is not None:
self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignid'])
self.params['stackid'] = self.alignstackdata['stack'].dbid
elif self.params['clusterid'] is not None:
self.clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
self.alignstackdata = self.clusterstackdata['clusterrun']['alignstack']
self.params['stackid'] = self.alignstackdata['stack'].dbid
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
#if self.params['description'] is None:
# apDisplay.printError("run description was not defined")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if not self.params['fastmode'] in self.fastmodes:
apDisplay.printError("fast mode must be on of: "+str(self.fastmodes))
maxparticles = 150000
minparticles = 50
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
if self.params['numpart'] < minparticles:
apDisplay.printError("not enough particles requested, min: "
+ str(minparticles) + " requested: " + str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
### find number of processors
if self.params['nproc'] is None:
self.nproc = apParam.getNumProcessors()
else:
self.nproc = self.params['nproc']
self.mpirun = self.checkMPI()
def checkConflicts(self):
apDisplay.printMsg("/bin/rm -f *.hed *.img *.doc *.vol *.mrc *.spi *.log *~ *.pickle *.hist *.proj *.xmp *.sel *.basis *.original; rm -fr volume*")
### check for missing and duplicate entries
#if self.params['alignid'] is None and self.params['clusterid'] is None:
# apDisplay.printError("Please provide either --cluster-id or --align-id")
if self.params['alignid'] is not None and self.params['clusterid'] is not None:
apDisplay.printError("Please provide only one of either --cluster-id or --align-id")
if self.params['modelstr'] is None and self.params['nvol'] is None:
apDisplay.printError("Please provide model numbers or number of volumes")
elif self.params['modelstr'] is not None:
modellist = self.params['modelstr'].split(",")
self.params['modelids'] = []
for modelid in modellist:
self.params['modelids'].append(int(modelid))
### get the stack ID from the other IDs
if self.params['alignid'] is not None:
self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignid'])
self.params['stackid'] = self.alignstackdata['stack'].dbid
elif self.params['clusterid'] is not None:
self.clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
self.alignstackdata = self.clusterstackdata['clusterrun']['alignstack']
self.params['stackid'] = self.alignstackdata['stack'].dbid
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
#if self.params['description'] is None:
# apDisplay.printError("run description was not defined")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if not self.params['fastmode'] in self.fastmodes:
apDisplay.printError("fast mode must be on of: "+str(self.fastmodes))
maxparticles = 150000
minparticles = 50
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
if self.params['numpart'] < minparticles:
apDisplay.printError("not enough particles requested, min: "
+ str(minparticles) + " requested: " + str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
### find number of processors
if self.params['nproc'] is None:
self.nproc = apParam.getNumProcessors()
else:
self.nproc = self.params['nproc']
self.mpirun = self.checkMPI()
def checkConflicts(self):
if self.params["stackid"] is None:
apDisplay.printError("stack id was not defined")
if self.params["modelid"] is None:
apDisplay.printError("model id was not defined")
if self.params["runname"] is None:
apDisplay.printError("missing a reconstruction name")
if self.params["last"] is None:
self.params["last"] = apStack.getNumberStackParticlesFromId(self.params["stackid"])
self.boxsize = apStack.getStackBoxsize(self.params["stackid"], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
if self.params["reconstackid"] is not None:
reconboxsize = apStack.getStackBoxsize(self.params["reconstackid"], msg=False)
reconapix = apStack.getStackPixelSizeFromStackId(self.params["reconstackid"])
refinenumpart = apStack.getNumberStackParticlesFromId(self.params["stackid"])
reconnumpart = apStack.getNumberStackParticlesFromId(self.params["reconstackid"])
if reconboxsize != self.boxsize:
apDisplay.printError("Boxsize do not match for stacks")
if reconapix != self.apix:
apDisplay.printError("Pixelsize do not match for stacks")
if refinenumpart != reconnumpart:
apDisplay.printError("Number of particles do not match for stacks")
maxmask = math.floor(self.apix * (self.boxsize - 10) / 2.0)
if self.params["mask"] is None:
apDisplay.printWarning("mask was not defined, setting to boxsize: %d" % (maxmask))
self.params["mask"] = maxmask
if self.params["mask"] > maxmask:
apDisplay.printWarning("mask was too big, setting to boxsize: %d" % (maxmask))
self.params["mask"] = maxmask
if self.params["noctf"] is True:
apDisplay.printWarning("Using no CTF method")
self.params["wgh"] = -1.0
if self.params["nodes"] > 1 and self.params["cluster"] is False:
apDisplay.printError("cluster mode must be enabled to run on more than 1 node")
if self.params["ppn"] is None:
if self.params["cluster"] is True:
apDisplay.printError("you must define ppn for cluster mode")
self.params["ppn"] = apParam.getNumProcessors()
apDisplay.printMsg("Setting number of processors to %d" % (self.params["ppn"]))
if self.params["rpn"] is None:
self.params["rpn"] = self.params["ppn"]
self.params["nproc"] = self.params["nodes"] * self.params["rpn"]
### get the symmetry data
if self.params["sym"] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params["sym"])
self.params["symm_id"] = self.symmdata.dbid
self.params["symm_name"] = self.symmdata["eman_name"]
apDisplay.printMsg("Selected symmetry %s with id %s" % (self.symmdata["eman_name"], self.symmdata.dbid))
### set cs value
self.params["cs"] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['runname'] is None:
apDisplay.printError("missing a reconstruction name")
if self.params['last'] is None:
self.params['last'] = apStack.getNumberStackParticlesFromId(self.params['stackid'])
self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
if self.params['reconstackid'] is not None:
reconboxsize = apStack.getStackBoxsize(self.params['reconstackid'], msg=False)
reconapix = apStack.getStackPixelSizeFromStackId(self.params['reconstackid'])
refinenumpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
reconnumpart = apStack.getNumberStackParticlesFromId(self.params['reconstackid'])
if reconboxsize != self.boxsize:
apDisplay.printError("Boxsize do not match for stacks")
if reconapix != self.apix:
apDisplay.printError("Pixelsize do not match for stacks")
if refinenumpart != reconnumpart:
apDisplay.printError("Number of particles do not match for stacks")
maxmask = math.floor(self.apix*(self.boxsize-10)/2.0)
if self.params['mask'] is None:
apDisplay.printWarning("mask was not defined, setting to boxsize: %d"%(maxmask))
self.params['mask'] = maxmask
if self.params['mask'] > maxmask:
apDisplay.printWarning("mask was too big, setting to boxsize: %d"%(maxmask))
self.params['mask'] = maxmask
if self.params['noctf'] is True:
apDisplay.printWarning("Using no CTF method")
self.params['wgh'] = -1.0
if self.params['nodes'] > 1 and self.params['cluster'] is False:
apDisplay.printError("cluster mode must be enabled to run on more than 1 node")
if self.params['ppn'] is None:
if self.params['cluster'] is True:
apDisplay.printError("you must define ppn for cluster mode")
self.params['ppn'] = apParam.getNumProcessors()
apDisplay.printMsg("Setting number of processors to %d"%(self.params['ppn']))
if self.params['rpn'] is None:
self.params['rpn'] = self.params['ppn']
self.params['nproc'] = self.params['nodes']*self.params['rpn']
### get the symmetry data
if self.params['sym'] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params['sym'])
self.params['symm_id'] = self.symmdata.dbid
self.params['symm_name'] = self.symmdata['eman_name']
apDisplay.printMsg("Selected symmetry %s with id %s"%(self.symmdata['eman_name'], self.symmdata.dbid))
### set cs value
self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self):
if self.params["thresh"] is None:
apDisplay.printError("threshold was not defined")
### Check if we have templates
if self.params["templateliststr"] is None:
apDisplay.printError("template list was not specified, e.g. --template-list=34,56,12")
### Parse template list
oldtemplateids = self.params["templateliststr"].split(",")
self.params["templateIds"] = []
for tid in oldtemplateids:
templateid = abs(int(tid))
self.params["templateIds"].append(templateid)
if self.params["templatemirrors"] is True:
self.params["templateIds"].append(-1 * templateid)
### Check if we have ranges
if self.params["rangeliststr"] is None:
apDisplay.printError("range not specified, please provide range in the order of templateIds")
### Check that numbers of ranges and templates are equal
rangestrlist = self.params["rangeliststr"].split("x")
if len(oldtemplateids) != len(rangestrlist):
apDisplay.printError("the number of templates and ranges do not match")
### Parse range list
i = 0
for rangestr in rangestrlist:
self.params["range" + str(i)] = rangestr
angs = rangestr.split(",")
if not len(angs) == 3:
apDisplay.printError("the range is not defined correctly")
self.params["startang" + str(i + 1)] = float(angs[0])
self.params["endang" + str(i + 1)] = float(angs[1])
self.params["incrang" + str(i + 1)] = float(angs[2])
self.params["mirror" + str(i + 1)] = False
if self.params["templatemirrors"] is True:
i += 1
self.params["startang" + str(i + 1)] = float(angs[0])
self.params["endang" + str(i + 1)] = float(angs[1])
self.params["incrang" + str(i + 1)] = float(angs[2])
self.params["mirror" + str(i + 1)] = True
i += 1
### get number of processors:
nproc = apParam.getNumProcessors()
if not self.params["nproc"]:
self.params["nproc"] = nproc
elif nproc < self.params["nproc"]:
apDisplay.printWarning("Limiting number of processors to the %i that are available" % nproc)
self.params["nproc"] = nproc
return
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['numrefs'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
self.stackdata = apStack.getOnlyStackData(self.params['stackid'],
msg=False)
stackfile = os.path.join(self.stackdata['path']['path'],
self.stackdata['name'])
# check for virtual stack
self.params['virtualdata'] = None
if not os.path.isfile(stackfile):
vstackdata = apStack.getVirtualStackParticlesFromId(
self.params['stackid'])
npart = len(vstackdata['particles'])
self.params['virtualdata'] = vstackdata
else:
npart = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > npart:
apDisplay.printError("trying to use more particles " +
str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.clipsize = int(math.floor(
boxsize / float(self.params['bin'] * 2))) * 2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d" %
(self.params['clipsize'], self.clipsize))
self.clipsize = self.params['clipsize']
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
self.mpirun = self.checkMPI()
if self.mpirun is None:
apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
if self.params['nproc'] < 2:
apDisplay.printError(
"Only the MPI version of CL2D is currently supported, must run with > 1 CPU"
)
def start(self):
"""
This is the core of your function.
You decide what happens here!
"""
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
original_stackfile = os.path.join(stackdata['path']['path'],
stackdata['name'])
filtered_stackfile = os.path.join(self.params['rundir'],
self.timestamp + ".hed")
apFile.removeStack(filtered_stackfile, warn=False)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
boxsize = apStack.getStackBoxsize(self.params['stackid'])
emancmd = "proc2d %s %s apix=%.3f " % (original_stackfile,
filtered_stackfile, apix)
if self.params['lowpass'] is not None:
emancmd += " lp=%.3f " % (self.params['lowpass'])
if self.params['highpass'] is not None:
emancmd += " hp=%.3f " % (self.params['highpass'])
if self.params['bin'] is not None and self.params['bin'] > 1:
## determine a multiple of the bin that is divisible by 2 and less than the boxsize
clipsize = int(math.floor(boxsize / float(
self.params['bin'] * 2))) * 2 * self.params['bin']
emancmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
emancmd += " last=%d " % (self.params['numpart'] - 1)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
### confirm that it worked
if self.params['numpart'] != apFile.numImagesInStack(
filtered_stackfile):
apDisplay.printError("Missing particles in stack")
nproc = apParam.getNumProcessors()
### run the refine2d.py
emancmd = ("refine2d.py --iter=%d --ninitcls=%d --proc=%d %s" % (
self.params['numiter'],
self.params['numclasses'],
nproc,
filtered_stackfile,
))
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
self.commitToDatabase()
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['description'] is None:
apDisplay.printError("run description was not defined")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if self.params['partrad'] is None:
apDisplay.printError("particle radius was not defined")
### deal with particles
maxparticles = 150000
self.stackdata = apStack.getOnlyStackData(self.params['stackid'],
msg=False)
stackfile = os.path.join(self.stackdata['path']['path'],
self.stackdata['name'])
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles " +
str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(stackfile)))
### get num processors
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
### convert / check template data
if self.params['templatelist'] is None:
apDisplay.printError("template list was not provided")
self.templatelist = self.params['templatelist'].strip().split(",")
if not self.templatelist or type(self.templatelist) != type([]):
apDisplay.printError("could not parse template list=" +
self.params['templatelist'])
self.params['numtemplate'] = len(self.templatelist)
apDisplay.printMsg("Found " + str(self.params['numtemplate']) +
" templates")
### convert / check template data
if self.params['orientref'] is None:
apDisplay.printError("reference for orientation was not provided")
def fermiLowPassFilter(imgarray, pixrad=2.0, dataext="spi", nproc=None):
if dataext[0] == '.': dataext = dataext[1:]
if nproc is None:
nproc = apParam.getNumProcessors(msg=False)
### save array to spider file
spider.write(imgarray, "rawimg."+dataext)
### run the filter
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, nproc=nproc)
### filter request: infile, outfile, filter-type, inv-radius, temperature
mySpider.toSpiderQuiet("FQ", "rawimg", "filtimg", "5", str(1.0/pixrad), "0.04")
mySpider.close()
### read array from spider file
filtarray = spider.read("filtimg."+dataext)
### clean up
apFile.removeFile("rawimg."+dataext)
apFile.removeFile("filtimg."+dataext)
return filtarray
def backprojectRP(stackfile, eulerdocfile, volfile, pixrad, classnum, lambDa, numpart=None, selfile=None, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
### setup
starttime = time.time()
stackfile = spyder.fileFilter(stackfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
if selfile is not None:
selfile = spyder.fileFilter(selfile)
volfile = spyder.fileFilter(volfile)
counter = spyder.fileFilter("numiter")
if not os.path.isfile(stackfile+dataext):
apDisplay.printError("stack file not found: "+stackfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(volfile+dataext)
if (numpart is not None):
selection = "1-"+str(numpart)
elif (selfile is not None):
selection = selfile
else:
apDisplay.printError("Partilce selection is invalid for BP RP. Please make sure either numpart or selfile is specified in the function backprojectRP!")
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("BP RP x11",
stackfile+"@*****", #stack file
selection, #selection file or particle range
str(pixrad), #particle radius
eulerdocfile, #angle doc file
"*", #has symmetry?, does not work
volfile, #filename for volume
"%.1e,%.1f" % (lambDa, 0.0), #lambda, correction
"%d,%d" % (50,1), #iteration limit, mode
"%d,%d" % (0,0), #minimum, maximum
"%.5f" % ((1/(1+6*lambDa))*0.999), #smoothing constant = (1/(1+6*lambda))*0.999
)
mySpider.toSpider("SD 1,x11", str(classnum)+"/"+str(counter))
mySpider.close()
apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def fermiLowPassFilter(imgarray, pixrad=2.0, dataext="spi", nproc=None):
if dataext[0] == '.': dataext = dataext[1:]
if nproc is None:
nproc = apParam.getNumProcessors(msg=False)
### save array to spider file
spider.write(imgarray, "rawimg." + dataext)
### run the filter
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, nproc=nproc)
### filter request: infile, outfile, filter-type, inv-radius, temperature
mySpider.toSpiderQuiet("FQ", "rawimg", "filtimg", "5", str(1.0 / pixrad),
"0.04")
mySpider.close()
### read array from spider file
filtarray = spider.read("filtimg." + dataext)
### clean up
apFile.removeFile("rawimg." + dataext)
apFile.removeFile("filtimg." + dataext)
return filtarray
def estimateIterTime(self):
secperiter = 0.12037
### get num processors
if self.params['nproc'] is None:
nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
calctime = ((self.params['numpart'] / 1000.0) *
self.params['numtemplate'] *
(self.stack['boxsize'] / self.params['bin'])**2 /
self.params['psistep'] / float(nproc) * secperiter)
if self.params['mirror'] is True:
calctime *= 2.0
self.params['estimatedtime'] = calctime
apDisplay.printColor(
"Estimated first iteration time: " +
apDisplay.timeString(calctime), "purple")
def estimateIterTime(self):
secperiter = 0.12037
### get num processors
if self.params['nproc'] is None:
nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
calctime = (
(self.params['numpart']/1000.0)
*self.params['numrefs']
*(self.stack['boxsize']/self.params['bin'])**2
/self.params['psistep']
/float(nproc)
*secperiter
)
if self.params['mirror'] is True:
calctime *= 2.0
self.params['estimatedtime'] = calctime
apDisplay.printColor("Estimated first iteration time: "+apDisplay.timeString(calctime), "purple")
def threadCommands(commandlist, nproc=None, pausetime=1.0):
"""
takes a list of commands and runs N of them at a time
waits for one to finish and submits another thread
caveats:
only work on a single machine
"""
### set number of processes at a time
if nproc is None:
nproc = apParam.getNumProcessors()
if len(commandlist) < nproc:
nproc = len(commandlist)
### copy the list so we don't modify the original
localcmdlist = commandlist
joblist = []
### initialize threads
for i in range(nproc):
cmd = localcmdlist.pop(0)
writeThreadLog(cmd)
job = AppionJob(cmd)
joblist.append(job)
job.run()
### continue until no commands are left
sys.stderr.write("threading commands")
while len(localcmdlist) > 0 or len(joblist) > 0:
for job in joblist:
if job.poll() is not None:
joblist.remove(job)
if len(localcmdlist) > 0:
cmd = localcmdlist.pop(0)
writeThreadLog(cmd)
job = AppionJob(cmd)
joblist.append(job)
job.run()
sys.stderr.write(".")
time.sleep(pausetime)
#print joblist
sys.stderr.write("\n")
def threadCommands(commandlist, nproc=None, pausetime=1.0):
"""
takes a list of commands and runs N of them at a time
waits for one to finish and submits another thread
caveats:
only work on a single machine
"""
### set number of processes at a time
if nproc is None:
nproc = apParam.getNumProcessors()
if len(commandlist) < nproc:
nproc = len(commandlist)
### copy the list so we don't modify the original
localcmdlist = commandlist
joblist = []
### initialize threads
for i in range(nproc):
cmd = localcmdlist.pop(0)
writeThreadLog(cmd)
job = AppionJob(cmd)
joblist.append(job)
job.run()
### continue until no commands are left
sys.stderr.write("threading commands")
while len(localcmdlist) > 0 or len(joblist) > 0:
for job in joblist:
if job.poll() is not None:
joblist.remove(job)
if len(localcmdlist) > 0:
cmd = localcmdlist.pop(0)
writeThreadLog(cmd)
job = AppionJob(cmd)
joblist.append(job)
job.run()
sys.stderr.write(".")
time.sleep(pausetime)
#print joblist
sys.stderr.write("\n")
def checkConflicts(self):
if self.params['lowpass'] > 0:
apDisplay.printWarning("lowpass filter value greater than zero; will ignore")
if self.params['highpass'] > 0:
apDisplay.printWarning("highpass filter value greater than zero; will ignore")
self.params['highpass'] = None
self.params['lowpass'] = None
#if self.params['kfactor'] is not None and self.params['numslices'] is not None:
# apDisplay.printError("only one of 'kfactor' or 'numslices' can be defined")
if self.params['numslices'] is not None and self.params['numslices'] >= 15:
apDisplay.printError("too many slices defined by numslices, should be more like 2-6")
if self.params['sizerange'] is not None and self.params['sizerange'] >= 1.95*self.params['diam']:
apDisplay.printError("size range has be less than twice the diameter")
### get number of processors:
nproc = apParam.getNumProcessors()
if not self.params['nproc']:
self.params['nproc'] = nproc
elif nproc < self.params['nproc']:
apDisplay.printWarning("Limiting number of processors to the %i that are available"%nproc)
self.params['nproc'] = nproc
return
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['description'] is None:
apDisplay.printError("run description was not defined")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if self.params['partrad'] is None:
apDisplay.printError("particle radius was not defined")
### deal with particles
maxparticles = 150000
self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "+str(maxparticles)+" requested: "+str(self.params['numpart']))
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
### get num processors
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
### convert / check template data
if self.params['templatelist'] is None:
apDisplay.printError("template list was not provided")
self.templatelist = self.params['templatelist'].strip().split(",")
if not self.templatelist or type(self.templatelist) != type([]):
apDisplay.printError("could not parse template list="+self.params['templatelist'])
self.params['numtemplate'] = len(self.templatelist)
apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates")
### convert / check template data
if self.params['orientref'] is None:
apDisplay.printError("reference for orientation was not provided")
def start(self):
"""
This is the core of your function.
You decide what happens here!
"""
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp+".hed")
apFile.removeStack(filtered_stackfile, warn=False)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
boxsize = apStack.getStackBoxsize(self.params['stackid'])
emancmd = "proc2d %s %s apix=%.3f "%(original_stackfile, filtered_stackfile, apix)
if self.params['lowpass'] is not None:
emancmd += " lp=%.3f "%(self.params['lowpass'])
if self.params['highpass'] is not None:
emancmd += " hp=%.3f "%(self.params['highpass'])
if self.params['bin'] is not None and self.params['bin'] > 1:
## determine a multiple of the bin that is divisible by 2 and less than the boxsize
clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2*self.params['bin']
emancmd += " shrink=%d clip=%d,%d "%(self.params['bin'], clipsize, clipsize)
emancmd += " last=%d "%(self.params['numpart']-1)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
### confirm that it worked
if self.params['numpart'] != apFile.numImagesInStack(filtered_stackfile):
apDisplay.printError("Missing particles in stack")
nproc = apParam.getNumProcessors()
### run the refine2d.py
emancmd = ("refine2d.py --iter=%d --ninitcls=%d --proc=%d %s"
%(self.params['numiter'], self.params['numclasses'],
nproc, filtered_stackfile,)
)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
self.commitToDatabase()
def checkConflicts(self):
""" basic error handlign for running scripts """
### check for IMAGIC installation
try:
self.imagicroot = apIMAGIC.checkImagicExecutablePath()
except:
c = "some IMAGIC functions, e.g. prealignment and angular reconstitution "
c+= "may not work. Use EMAN instead"
apDisplay.printWarning(c)
self.imagicroot = None
self.params['useEMAN1'] is True
### check class averages
if self.params['templatestackid'] is not None:
self.stackdata = appiondata.ApTemplateStackData.direct_query(self.params['templatestackid'])
self.clsname = self.stackdata['templatename']
self.params['apix'] = self.stackdata['apix']
self.params['boxsize'] = self.stackdata['boxsize']
self.params['oldavgs'] = os.path.join(self.stackdata['path']['path'], self.clsname[:-4]+".hed")
self.params['avgs'] = os.path.basename(self.clsname)
elif self.params['clusterid'] is not None:
self.stackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
self.clsname = self.stackdata['avg_imagicfile']
self.params['apix'] = self.stackdata['clusterrun']['pixelsize']
self.params['boxsize'] = self.stackdata['clusterrun']['boxsize']
self.params['oldavgs'] = os.path.join(self.stackdata['path']['path'], self.clsname[:-4]+".hed")
self.params['avgs'] = self.clsname
elif self.params['classavgs'] is not None:
if self.params['apix'] is None:
apDisplay.printError("enter pixel size for manually uploaded classes")
self.params['oldavgs'] = self.params['classavgs']
bavgs = os.path.basename(self.params['classavgs'])
self.params['avgs'] = bavgs
else:
apDisplay.printError("enter class averages for the run")
if not os.path.isfile(os.path.abspath(self.params['oldavgs'])):
apDisplay.printError("cannot find input class averages")
if self.params['templatestackid'] is not None and self.params['clusterid'] is not None:
apDisplay.printError("enter either templatestack ID OR cluster ID")
if self.params['templatestackid'] is not None and self.params['classavgs'] is not None:
apDisplay.printError("enter either templatestack ID OR manually uploaded classes")
if self.params['clusterid'] is not None and self.params['classavgs'] is not None:
apDisplay.printError("enter either cluster ID OR manually uploaded classes")
### pretreatment
if self.imagicroot is not None and self.params['prealign'] is True:
warning = "particles will not be prealigned, IMAGIC is not installed or path "
warning+= "to IMAGIC cannot be located"
apDisplay.printWarning(warning)
### basic input parameters
self.params['numpart'] = apFile.numImagesInStack(self.params['oldavgs'])
if self.params['numpart'] < 3:
apDisplay.printError("need at least 3 class averages in stack to run")
self.params['boxsize'] = apFile.getBoxSize(self.params['oldavgs'])[0]
if self.params['num_volumes'] is None:
apDisplay.printError("specify the number of volumes to produce")
if self.params['apix'] is None:
apDisplay.printError("enter pixel size of class averages")
### refinement class averages
if self.params['r_templatestackid'] is not None:
self.rstackdata = appiondata.ApTemplateStackData.direct_query(self.params['r_templatestackid'])
self.rclsname = self.rstackdata['templatename']
self.params['refineapix'] = self.rstackdata['apix']
self.params['refineboxsize'] = self.rstackdata['boxsize']
self.params['oldravgs'] = os.path.join(self.rstackdata['path']['path'], self.rclsname[:-4]+".hed")
self.params['ravgs'] = os.path.basename(self.rclsname)
elif self.params['r_clusterid'] is not None:
self.rstackdata = appiondata.ApClusteringStackData.direct_query(self.params['r_clusterid'])
self.rclsname = self.rstackdata['avg_imagicfile']
self.params['refineapix'] = self.rstackdata['clusterrun']['pixelsize']
self.params['refineboxsize'] = self.rstackdata['clusterrun']['boxsize']
self.params['oldravgs'] = os.path.join(self.rstackdata['path']['path'], self.rclsname[:-4]+".hed")
self.params['ravgs'] = self.rclsname
elif self.params['refine_classavgs'] is not None:
rbavgs = os.path.basename(self.params['refine_classavgs'])
self.params['ravgs'] = rbavgs
self.params['refineboxsize'] = apFile.getBoxSize(self.params['oldravgs'])[0]
if self.params['refineapix'] is None:
warning = "refinement pixel size not specified ... assuming it is the same"
apDisplay.printWarning(warning)
self.params['refineapix'] = self.params['apix']
else:
# self.params['ravgs'] = None
self.params['oldravgs'] = self.params['avgs']
self.params['ravgs'] = self.params['avgs']
self.params['refineapix'] = self.params['apix']
self.params['refineboxsize'] = self.params['boxsize']
if not os.path.isfile(os.path.abspath(self.params['oldravgs'])):
warning = "cannot find class averages for refinement using the "
warning+= "specified path. Original averages will be used instead."
apDisplay.printWarning(warning)
self.params['oldravgs'] = self.params['avgs']
self.params['ravgs'] = self.params['avgs']
### check for scaling
if self.params['scale'] is True:
self.scalefactor = float(64.0 / self.params['boxsize'])
self.params['apix'] = self.params['apix'] / self.scalefactor
self.params['boxsize'] = 64
else:
self.scalefactor = 1
### angular reconstitution checks
if self.params['keep_ordered'] < 1.0: ### probably specified as a fraction
self.params['keep_ordered'] = self.params['keep_ordered'] * 100 ### convert to percentage
self.params['keep_ordered_num'] = self.params['numpart'] * self.params['keep_ordered'] / 100
### EMAN1 cross common lines checks
if self.params['useEMAN1'] is True:
if self.params['images_per_volume'] is None:
self.params['images_per_volume'] = int(self.params['numpart'] * 0.66)
if self.params['images_per_volume'] > self.params['numpart']:
warning = "number of images per volume greater than number of class averages "
warning+= "in input. Setting to (0.66)*(num_averages)"
apDisplay.printWarning(warning)
self.params['images_per_volume'] = int(self.params['numpart'] * 0.66)
if self.params['images_per_volume'] > self.params['numpart'] * 0.75:
warning = "consider asking for less images per volume to provide more variety "
warning = "during common lines search"
apDisplay.printWarning(warning)
### number of processors for threading ONLY works on a single node
self.params['threadnproc'] = apParam.getNumProcessors()
### refinement parameters
if self.params['linmask'] == 0:
self.params['linmask'] = (self.params['boxsize']-2)/2*self.params['apix']
if self.params['mask_radius'] is None and self.params['linmask'] != 0:
self.params['mask_radius'] = self.params['linmask'] * 1.2
elif self.params['mask_radius'] is None and self.params['linmask'] == 0:
self.params['mask_radius'] = self.params['boxsize'] * self.params['refineapix']
if self.params['outer_radius'] is None:
self.params['outer_radius'] = self.params['mask_radius'] * 0.8
return
def start(self):
if self.params['method'] == 'any':
### startAny uses class averages
clusterstack, numimages = self.getClusterStack()
else:
### starticos, startoct, startcsym uses individual particles
clusterstack, numimages = self.getClusterParticles()
if self.params['method'] != 'any':
if self.params['numkeep'] is not None and numimages/10 < int(self.params['numkeep']):
apDisplay.printWarning("particle number of "+ self.params['numkeep']
+ " is greater than 10% of the number of selected classes")
elif self.params['numkeep'] is None:
self.params['numkeep'] = int(math.floor(numimages/20.0))+1
apDisplay.printWarning("numkeep was not defined, using %d particles"%(self.params['numkeep']))
nproc = apParam.getNumProcessors()
#construct command for each of the EMAN commonline method
if self.params['method'] == 'any':
startcmd = "startAny "+clusterstack+" proc="+str(nproc)
startcmd +=" sym="+self.symmdata['eman_name']
if self.params['mask'] is not None:
startcmd +=" mask="+str(self.params['mask'])
else:
maskrad = math.floor(self.clusterstackdata['clusterrun']['boxsize']/2.0)
startcmd +=" mask=%d"%(maskrad)
if self.params['rounds'] is not None:
startcmd +=" rounds="+str(self.params['rounds'])
elif self.params['method'] == 'csym':
startcmd = "startcsym "+clusterstack+" "
startcmd +=" "+str(self.params['numkeep'])
startcmd +=" sym="+self.symmdata['eman_name']
if self.params['imask'] is not None:
startcmd +=" imask="+self.params['imask']
elif self.params['method'] == 'oct':
startcmd = "startoct "+clusterstack+" "
startcmd +=" "+str(self.params['numkeep'])
elif self.params['method'] == 'icos':
startcmd = "starticos "+clusterstack+" "
startcmd +=" "+str(self.params['numkeep'])
if self.params['imask'] is not None:
startcmd +=" imask="+self.params['imask']
apDisplay.printMsg("Creating 3D model with EMAN function: start"+self.params['method'])
apFile.removeFile("threed.0a.mrc")
apFile.removeFile("eman.log")
apEMAN.executeEmanCmd(startcmd, verbose=False, logfile="eman.log")
#apEMAN.executeEmanCmd(startcmd, verbose=True)
finalmodelname = "threed-%s-eman_start%s.mrc"%(self.timestamp, self.params['method'])
finalmodelpath = os.path.join(self.params['rundir'], finalmodelname)
apDisplay.printMsg("Final model name: "+finalmodelname)
finalmodel = "threed.0a.mrc"
if os.path.isfile(finalmodel):
emancmd = "proc3d %s %s norm=0,1 origin=0,0,0"%(finalmodel, finalmodelpath)
#shutil.move(finalmodel, finalmodelpath)
apEMAN.executeEmanCmd(emancmd, verbose=True)
if not apVolume.isValidVolume(finalmodelpath):
apDisplay.printError("Created volume is not valid")
else:
apDisplay.printError("No 3d model was created")
### upload it
self.uploadDensity(finalmodelpath)
### chimera imaging
apChimera.renderSnapshots(finalmodelpath, contour=self.params['contour'],
zoom=self.params['zoom'], sym=self.symmdata['eman_name'])
apChimera.renderAnimation(finalmodelpath, contour=self.params['contour'],
zoom=self.params['zoom'], sym=self.symmdata['eman_name'])
### remove stack
if apFile.stackSize(clusterstack)/1024**2 > 10:
### file bigger than 10MB
apFile.removeStack(clusterstack)
apFile.removeFile("threed.0a.mrc")
def alignStack(oldstack, alignedstack, partlist, dataext=".spi"):
"""
write aligned stack -- with python loop
inputs:
oldstack
newstack (empty)
list of particle dictionaries for operations
modifies:
newstack
output:
none
I tried this loop in both spider and python;
python was faster?!? -neil
"""
if not os.path.isfile(oldstack + dataext):
apDisplay.printError("Could not find original stack: " + oldstack +
dataext)
boxsize = apFile.getBoxSize(oldstack + dataext)
apDisplay.printMsg("applying alignment parameters to stack")
apFile.removeFile(alignedstack + dataext)
count = 0
t0 = time.time()
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext,
logo=True,
nproc=nproc,
log=False)
#create stack in core
numpart = len(partlist)
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d" % (boxsize), #boxsize
str(numpart + 1), #num part to create in memory
str(numpart + 1), #max particle number
)
for partdict in partlist:
partnum = partdict['num']
#if partdict['num'] in [3,6,7]:
# print partdict['num'], partdict['template'], partdict['mirror'], round(partdict['rot'],3)
### Rotate and Shift operations
count += 1
#rotate/shift
mySpider.toSpiderQuiet(
"RT SQ",
spyder.fileFilter(oldstack) + "@" + ("%06d" % (partnum)),
"_1",
str(partdict['rot']),
str(partdict['xshift']) + "," + str(partdict['yshift']),
)
#mirror, if necessary
if 'mirror' in partdict and partdict['mirror'] is True:
mySpider.toSpiderQuiet(
"MR",
"_1",
"_2@" + ("%06d" % (partnum)),
"Y",
)
else:
mySpider.toSpiderQuiet(
"CP",
"_1",
"_2@" + ("%06d" % (partnum)),
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP",
"_2@",
spyder.fileFilter(alignedstack) + "@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE",
"_2",
)
mySpider.close()
apDisplay.printMsg("Completed transforming %d particles in %s" %
(count, apDisplay.timeString(time.time() - t0)))
if count < 1:
apDisplay.printError("Failed to transform any particles")
if not os.path.isfile(alignedstack + dataext):
apDisplay.printError("Failed to create stack " + alignedstack +
dataext)
return
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local file
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
a.setValue('append',False)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert') is True
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
# clip not yet implemented
# self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### stack is needed by uploadMaxlike.py
#apFile.removeStack(self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -nref "+str(self.params['numrefs'])
+" -iter "+str(self.params['maxiter'])
+" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp)
+" -psi_step "+str(self.params['psistep'])
)
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-17 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### use student's T distribution
if self.params['student'] is True:
xmippopts += " -student "
### write cluster job file
if self.params['cluster'] is True:
self.writeClusterJobFile()
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
self.estimateIterTime()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using "+str(nproc)+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun+" -np "+str(nproc)+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe+" "+xmippopts
self.writeXmippLog(xmippcmd)
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
def checkConflicts(self):
### first get all stack data
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
### modify boxsize and pixelsize according to binning
self.boxsize = self.stack['boxsize']
self.clipsize = int(math.floor(self.boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
self.apix = self.stack['apix']
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize)*self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
self.boxsize = clipsize
self.apix = self.apix * self.params['bin']
### basic error checking
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['ncls'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 1000000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > 5000:
apDisplay.printWarning("initial model calculation may not work with less than 5000 particles")
self.mpirun = self.checkMPI()
# if self.mpirun is None:
# apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
### SIMPLE defaults and error checking
if self.params['ring2'] is None:
self.params['ring2'] = (self.boxsize/2) - 2
if self.params['ncls'] > 2000:
apDisplay.printError("number of classes should be less than 2000 for subsequent ORIGAMI run to work")
if self.params['ncls'] > self.params['numpart']:
self.params['ncls'] = self.params['numpart'] / self.params['minp']
if self.params['mask'] is None:
self.params['mask'] = (self.boxsize/2) - 2
if self.params['mw'] is None:
apDisplay.printError("please specify the molecular weight (in kDa)")
def uploadScriptData(self):
"""
Using tables to track program run parameters in a generic fashion
inspired by Roberto Marabini and Carlos Oscar Sanchez Sorzano from the Xmipp team/Carazo lab
"""
apDisplay.printMsg("Uploading ScriptData....")
prognameq = appiondata.ScriptProgramName()
prognameq['name'] = self.functionname
userq = appiondata.ScriptUserName()
userdict = apParam.getUserDict()
if userdict:
userq['name'] = userdict['username']
userq['uid'] = userdict['uid']
userq['gid'] = userdict['gid']
userq['fullname'] = userdict['fullname']
unixshell = userdict['unixshell']
else:
userq['name'] = "unknown"
unixshell = None
hostq = appiondata.ScriptHostName()
hostq['name'] = apParam.getHostname()
hostq['ip'] = apParam.getHostIP()
hostq['system'] = apParam.getSystemName()
hostq['distro'] = apParam.getLinuxDistro()
hostq['nproc'] = apParam.getNumProcessors()
hostq['memory'] = apParam.getTotalMemory()
hostq['cpu_vendor'] = apParam.getCPUVendor()
hostq['gpu_vendor'] = apParam.getGPUVendor()
hostq['arch'] = apParam.getMachineArch()
progrunq = appiondata.ScriptProgramRun()
progrunq['runname'] = self.params['runname']
progrunq['progname'] = prognameq
progrunq['username'] = userq
progrunq['hostname'] = hostq
progrunq['unixshell'] = unixshell
progrunq['rundir'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
progrunq['job'] = self.getClusterJobData()
appiondir = apParam.getAppionDirectory()
### get appion version/subversion revision
progrunq['revision'] = None
versionfile = os.path.join(appiondir, "appionlib/version.txt")
if os.path.isfile(versionfile):
f = open(versionfile, 'r')
line = f.readline()
f.close()
sline = line.strip()
progrunq['revision'] = sline
if os.path.isdir(os.path.join(appiondir, ".svn")):
if progrunq['revision'] is None:
progrunq['revision'] = version.getSubversionRevision(appiondir)
else:
progrunq['revision'] += "-" + version.getSubversionRevision(
appiondir)
if not progrunq['revision']:
progrunq['revision'] = 'unknown'
apDisplay.printMsg("Running Appion version '%s'" %
(progrunq['revision']))
progrunq['appion_path'] = appiondata.ApPathData(
path=os.path.abspath(appiondir))
for paramname in self.params.keys():
paramnameq = appiondata.ScriptParamName()
paramnameq['name'] = paramname
paramnameq['progname'] = prognameq
paramvalueq = appiondata.ScriptParamValue()
paramvalueq['value'] = str(self.params[paramname])
usage = self.usageFromParamDest(paramname, self.params[paramname])
#print "usage: ", usage
paramvalueq['usage'] = usage
paramvalueq['paramname'] = paramnameq
paramvalueq['progrun'] = progrunq
if usage is not None:
paramvalueq.insert()
def uploadScriptData(self):
"""
Using tables to track program run parameters in a generic fashion
inspired by Roberto Marabini and Carlos Oscar Sanchez Sorzano from the Xmipp team/Carazo lab
"""
apDisplay.printMsg("Uploading ScriptData....")
prognameq = appiondata.ScriptProgramName()
prognameq['name'] = self.functionname
userq = appiondata.ScriptUserName()
userdict = apParam.getUserDict()
if userdict:
userq['name'] = userdict['username']
userq['uid'] = userdict['uid']
userq['gid'] = userdict['gid']
userq['fullname'] = userdict['fullname']
unixshell = userdict['unixshell']
else:
userq['name'] = "unknown"
unixshell = None
hostq = appiondata.ScriptHostName()
hostq['name'] = apParam.getHostname()
hostq['ip'] = apParam.getHostIP()
hostq['system'] = apParam.getSystemName()
hostq['distro'] = apParam.getLinuxDistro()
hostq['nproc'] = apParam.getNumProcessors()
hostq['memory'] = apParam.getTotalMemory()
hostq['cpu_vendor'] = apParam.getCPUVendor()
hostq['gpu_vendor'] = apParam.getGPUVendor()
hostq['arch'] = apParam.getMachineArch()
progrunq = appiondata.ScriptProgramRun()
progrunq['runname'] = self.params['runname']
progrunq['progname'] = prognameq
progrunq['username'] = userq
progrunq['hostname'] = hostq
progrunq['unixshell'] = unixshell
progrunq['rundir'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
progrunq['job'] = self.getClusterJobData()
appiondir = apParam.getAppionDirectory()
### get appion version/subversion revision
progrunq['revision'] = None
versionfile = os.path.join(appiondir, "appionlib/version.txt")
if os.path.isfile(versionfile):
f = open(versionfile, 'r')
line = f.readline()
f.close()
sline = line.strip()
progrunq['revision'] = sline
if os.path.isdir(os.path.join(appiondir, ".svn")):
if progrunq['revision'] is None:
progrunq['revision'] = version.getSubversionRevision(appiondir)
else:
progrunq['revision'] += "-"+version.getSubversionRevision(appiondir)
if not progrunq['revision']:
progrunq['revision'] = 'unknown'
apDisplay.printMsg("Running Appion version '%s'"%(progrunq['revision']))
progrunq['appion_path'] = appiondata.ApPathData(path=os.path.abspath(appiondir))
for paramname in self.params.keys():
paramnameq = appiondata.ScriptParamName()
paramnameq['name'] = paramname
paramnameq['progname'] = prognameq
paramvalueq = appiondata.ScriptParamValue()
paramvalueq['value'] = str(self.params[paramname])
usage = self.usageFromParamDest(paramname, self.params[paramname])
#print "usage: ", usage
paramvalueq['usage'] = usage
paramvalueq['paramname'] = paramnameq
paramvalueq['progrun'] = progrunq
if usage is not None:
paramvalueq.insert()
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local temp file
proccmd = "proc2d "+self.stack['file']+" temp.hed apix="+str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize)*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
proccmd += " last="+str(self.params['numpart']-1)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### process stack to final file
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
proccmd = "proc2d temp.hed "+self.params['localstack']+" apix="+str(self.stack['apix']*self.params['bin'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
apFile.removeStack("temp.hed")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### convert stack into single spider files
templateselfile = self.initializeTemplates()
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -iter "+str(self.params['maxiter'])
+" -ref "+templateselfile
+" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp)
+" -psi_step "+str(self.params['psistep'])
)
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-18 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using "+str(nproc-1)+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun+" -np "+str(nproc-1)+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe+" "+xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local temp file
proccmd = "proc2d " + self.stack['file'] + " temp.hed apix=" + str(
self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### process stack to final file
self.params['localstack'] = os.path.join(self.params['rundir'],
self.timestamp + ".hed")
proccmd = "proc2d temp.hed " + self.params[
'localstack'] + " apix=" + str(
self.stack['apix'] * self.params['bin'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
apFile.removeStack("temp.hed")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### convert stack into single spider files
templateselfile = self.initializeTemplates()
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -iter " + str(self.params['maxiter']) + " -ref " +
templateselfile + " -o " +
os.path.join(self.params['rundir'], "part" + self.timestamp) +
" -psi_step " + str(self.params['psistep']))
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-18 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using " + str(nproc - 1) + " processors!",
"green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun + " -np " + str(
nproc - 1) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
def crossCorrelateAndShift(infile, reffile, alignfile, ccdocfile, numpart, dataext=".spi"):
### rewriten to do the whole thing in memory in SPIDER, it should be faster
starttime = time.time()
infile = spyder.fileFilter(infile)
reffile = spyder.fileFilter(reffile)
alignfile = spyder.fileFilter(alignfile)
partimg = "_4"
ccmap = "_5"
windccmap = "_6"
boxsize = apFile.getBoxSize(infile+dataext)
if not os.path.isfile(infile+dataext):
apDisplay.printError("input stack file not found: "+infile+dataext)
if not os.path.isfile(reffile+dataext):
apDisplay.printError("reference stack file not found: "+reffile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
### Allocate empty stack
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d"%(boxsize), #boxsize
str(numpart+1), #num part to create in memory
str(numpart+1), #max particle number
)
partnum = 0
while partnum < numpart:
partnum+=1
mySpider.toSpiderQuiet("CP",
infile+("@%05d"%(partnum)), #picture
partimg,
)
### cross correlate images; reversed order to avoid -1*shift
mySpider.toSpiderQuiet("CC N",
reffile+("@%05d"%(partnum)), #reference
partimg, #picture
ccmap, #output file
)
### cannot shift more the 1/4 size of the image
mySpider.toSpiderQuiet("FI x52", partimg, "12" )
mySpider.toSpiderQuiet("x54=int(x52/2)") #window size
mySpider.toSpiderQuiet("x55=int(x52/4)") #window topleft
mySpider.toSpiderQuiet("WI",
ccmap, #input file
windccmap, #output file
"x54,x54", #window size
"x55,x55", #window origin
)
### find the cross-correlation peak
mySpider.toSpiderQuiet("x56=int(x52/4)+1") #center of window
mySpider.toSpiderQuiet("PK M x11,x12,x13,x14",
windccmap, #input ccmap file
"x56,x56", #origin coordinates
)
### save info to doc file
mySpider.toSpiderQuiet("SD %d,x13,x14"%(partnum),
ccdocfile, #input ccmap file
)
### shift the images images
mySpider.toSpiderQuiet("SH",
partimg, #old stack
("_2@%05d"%(partnum)), #new stack
"x13,x14", #shift value file
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP", "_2@",
alignfile+"@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE", "_2",
)
mySpider.close()
apDisplay.printColor("finished shifting particles in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def start(self):
self.runtime = 0
self.checkCoranRun()
### convert stack to spider
self.alignstackdata = self.getAlignedStack()
maskpixrad = self.params['maskrad'] / self.alignstackdata[
'pixelsize'] / self.params['bin']
boxpixdiam = int(math.ceil(maskpixrad) + 1) * 2
if boxpixdiam * self.params['bin'] > self.alignstackdata['boxsize']:
boxpixdiam = math.floor(self.alignstackdata['boxsize'] /
self.params['bin'])
clippixdiam = boxpixdiam * self.params['bin']
apDisplay.printMsg("Pixel mask radius=" + str(maskpixrad))
oldalignedstack = os.path.join(self.alignstackdata['path']['path'],
self.alignstackdata['imagicfile'])
alignedstackname = re.sub("\.", "_",
self.alignstackdata['imagicfile']) + ".spi"
alignedstack = os.path.join(self.params['rundir'], alignedstackname)
apFile.removeFile(alignedstack)
emancmd = ("proc2d %s %s spiderswap shrink=%d clip=%d,%d edgenorm" %
(oldalignedstack, alignedstack, self.params['bin'],
clippixdiam, clippixdiam))
if self.params['numpart'] is not None:
emancmd += " last=%d" % (self.params['numpart'] - 1)
numpart = self.params['numpart']
else:
numpart = self.getNumAlignedParticles()
apEMAN.executeEmanCmd(emancmd, verbose=True)
### get number of processors
nproc = apParam.getNumProcessors()
esttime = classification.estimateTime(numpart, maskpixrad)
apDisplay.printColor("Running spider this can take awhile, estimated time: "+\
apDisplay.timeString(esttime),"cyan")
### do correspondence analysis
corantime = time.time()
self.contriblist = classification.correspondenceAnalysis(
alignedstack,
boxsize=boxpixdiam,
maskpixrad=maskpixrad,
numpart=numpart,
numfactors=self.params['numfactors'],
nproc=nproc)
corantime = time.time() - corantime
### make dendrogram
dendrotime = time.time()
classification.makeDendrogram(numfactors=min(
3, self.params['numfactors']),
nproc=nproc)
dendrotime = time.time() - dendrotime
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = corantime
self.insertCoranRun(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apFile.removeFile(alignedstack, warn=True)
apDisplay.printMsg("Correspondence Analysis time: " +
apDisplay.timeString(corantime))
apDisplay.printMsg("Make Dendrogram time: " +
apDisplay.timeString(dendrotime))
apDisplay.printMsg("Database Insertion time: " +
apDisplay.timeString(inserttime))
def start(self):
self.insertTopolRepJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
self.dumpParameters()
self.params['canexe'] = self.getCANPath()
### process stack to local file
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
a.setValue('append',False)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
if self.params['premask'] is True and self.params['mramethod'] != 'imagic':
a.setValue('mask',self.params['mask'])
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
if self.params['uploadonly'] is not True:
if os.path.isfile(os.path.join(self.params['rundir'],"stack.hed")):
self.params['localstack']=os.path.join(self.params['rundir'],"stack.hed")
else:
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### IMAGIC mask particles before alignment
if self.params['premask'] is True and self.params['mramethod'] == 'imagic':
# convert mask to fraction for imagic
maskfrac = self.workingmask*2/self.workingboxsize
maskstack = imagicFilters.softMask(self.params['localstack'],mask=maskfrac)
shutil.move(maskstack+".hed",os.path.splitext(self.params['localstack'])[0]+".hed")
shutil.move(maskstack+".img",os.path.splitext(self.params['localstack'])[0]+".img")
origstack = self.params['localstack']
### find number of processors
# if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
if self.params['uploadonly'] is not True:
aligntime = time.time()
# run through iterations
for i in range(0,self.params['iter']+1):
# move back to starting directory
os.chdir(self.params['rundir'])
# set up next iteration directory
self.params['currentiter'] = i
self.params['iterdir'] = os.path.abspath("iter%02i" % i)
self.params['iterdir'] = string.replace(self.params['iterdir'],"/jetstor/APPION","")
if os.path.exists(self.params['iterdir']):
apDisplay.printError("Error: directory '%s' exists, aborting alignment" % self.params['iterdir'])
# create directory for iteration
os.makedirs(self.params['iterdir'])
os.chdir(self.params['iterdir'])
# if at first iteration, create initial class averages
if i == 0:
# first rewrite localstack headers if particles not pre-masked
if self.params['premask'] is False and self.params['mramethod'] == "imagic":
imagicFilters.takeoverHeaders(self.params['localstack'],self.params['numpart'],self.workingboxsize)
self.params['alignedstack'] = os.path.splitext(self.params['localstack'])[0]
if self.params['msamethod']=='imagic':
self.runIMAGICmsa()
else:
self.runCAN()
continue
# using references from last iteration, run multi-ref alignment
if self.params['mramethod'] == "imagic":
# rewrite class headers
imagicFilters.takeoverHeaders(self.params['currentcls'],self.params['currentnumclasses'],self.workingboxsize)
self.runIMAGICmra()
else:
self.runEMANmra()
# create class averages from aligned stack
if self.params['msamethod']=='imagic':
self.runIMAGICmsa()
else:
self.runCAN()
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
## set upload information params:
else:
## get last iteration
alliters = glob.glob("iter*")
alliters.sort()
## get iteration number from iter dir
self.params['currentiter'] = int(alliters[-1][-2:])
self.params['iterdir'] = os.path.join(self.params['rundir'],alliters[-1])
self.params['currentcls'] = "classes%02i"%(self.params['currentiter'])
## go into last iteration directory
os.chdir(self.params['iterdir'])
self.params['alignedstack'] = os.path.abspath("mrastack")
if os.path.isfile(os.path.join(self.params['rundir'],self.params['currentcls']+".hed")):
p1 = os.path.join(self.params['rundir'],self.params['currentcls'])
p2 = os.path.join(self.params['iterdir'],self.params['currentcls'])
shutil.move(p1+".hed",p2+".hed")
shutil.move(p1+".img",p2+".img")
## sort the class averages
self.sortClassAverages()
### get particle information from last iteration
if self.params['mramethod']=='imagic':
partlist = self.readPartIMAGICFile()
else:
partlist = self.readPartEMANFile()
if self.params['msamethod']=='imagic':
partrefdict = self.imagicClassificationToDict()
else:
partrefdict = self.canClassificationToDict()
# move back to starting directory
os.chdir(self.params['rundir'])
# move aligned stack to current directory for appionweb
if not os.path.isfile("mrastack.hed"):
shutil.move(self.params['alignedstack']+".hed","mrastack.hed")
shutil.move(self.params['alignedstack']+".img","mrastack.img")
# rewrite header
if self.params['mramethod'] == "imagic" or self.params['msamethod'] == 'imagic':
imagicFilters.takeoverHeaders("mrastack",self.params['numpart'],self.workingboxsize)
# move actual averages to current directory
if self.params['msamethod']=='can':
if not os.path.isfile("classes_avg.hed"):
self.applySort()
# save actual class averages as refs in database
self.params['currentcls']="classes_avg"
### create an average mrc of final references
if not os.path.isfile("average.mrc"):
apStack.averageStack(stack=self.params['currentcls']+".hed")
self.dumpParameters()
### remove the filtered stack
apFile.removeStack(origstack)
### save to database
if self.params['commit'] is True:
self.insertRunIntoDatabase()
self.insertParticlesIntoDatabase(partlist, partrefdict)
def alignStack(oldstack, alignedstack, partlist, dataext=".spi"):
"""
write aligned stack -- with python loop
inputs:
oldstack
newstack (empty)
list of particle dictionaries for operations
modifies:
newstack
output:
none
I tried this loop in both spider and python;
python was faster?!? -neil
"""
if not os.path.isfile(oldstack+dataext):
apDisplay.printError("Could not find original stack: "+oldstack+dataext)
boxsize = apFile.getBoxSize(oldstack+dataext)
apDisplay.printMsg("applying alignment parameters to stack")
apFile.removeFile(alignedstack+dataext)
count = 0
t0 = time.time()
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
#create stack in core
numpart = len(partlist)
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d"%(boxsize), #boxsize
str(numpart+1), #num part to create in memory
str(numpart+1), #max particle number
)
for partdict in partlist:
partnum = partdict['num']
#if partdict['num'] in [3,6,7]:
# print partdict['num'], partdict['template'], partdict['mirror'], round(partdict['rot'],3)
### Rotate and Shift operations
count += 1
#rotate/shift
mySpider.toSpiderQuiet(
"RT SQ",
spyder.fileFilter(oldstack)+"@"+("%06d" % (partnum)),
"_1",
str(partdict['rot']), str(partdict['xshift'])+","+str(partdict['yshift']),
)
#mirror, if necessary
if 'mirror' in partdict and partdict['mirror'] is True:
mySpider.toSpiderQuiet(
"MR", "_1",
"_2@"+("%06d" % (partnum)), "Y",
)
else:
mySpider.toSpiderQuiet(
"CP", "_1",
"_2@"+("%06d" % (partnum)),
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP", "_2@",
spyder.fileFilter(alignedstack)+"@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE", "_2",
)
mySpider.close()
apDisplay.printMsg("Completed transforming %d particles in %s"%(count, apDisplay.timeString(time.time()-t0)))
if count < 1:
apDisplay.printError("Failed to transform any particles")
if not os.path.isfile(alignedstack+dataext):
apDisplay.printError("Failed to create stack "+alignedstack+dataext)
return
def refBasedAlignParticles(stackfile,
templatestack,
origstackfile,
xysearch,
xystep,
numpart,
numtemplate,
firstring=2,
lastring=100,
dataext=".spi",
iternum=1,
oldpartlist=None):
"""
inputs:
stack
template
search params
outputs:
aligned stack
rotation/shift params
"""
### setup
if dataext in templatestack:
templatestack = templatestack[:-4]
if dataext in stackfile:
stackfile = stackfile[:-4]
if dataext in origstackfile:
origstackfile = origstackfile[:-4]
t0 = time.time()
rundir = "alignments"
apParam.createDirectory(rundir)
nproc = apParam.getNumProcessors()
### remove previous iterations
apFile.removeFile(rundir + "/paramdoc%02d%s" % (iternum, dataext))
### perform alignment, should I use 'AP SH' instead?
mySpider = spyder.SpiderSession(dataext=dataext,
logo=True,
nproc=nproc,
log=False)
mySpider.toSpider(
"AP MQ",
spyder.fileFilter(templatestack) + "@**", # reference image series
"1-" + str(numtemplate), # enter number of templates of doc file
str(int(xysearch)) + "," +
str(int(xystep)), # translation search range, step size
str(int(firstring)) + "," +
str(int(lastring)), # first and last ring for rotational correlation
spyder.fileFilter(stackfile) + "@******", # unaligned image series
"1-" + str(numpart), # enter number of particles of doc file
rundir + ("/paramdoc%02d" % (iternum)), # output angles document file
)
mySpider.close()
"""
### perform alignment, should I use 'AP SH' instead?
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("AP SH",
spyder.fileFilter(templatestack)+"@**", # reference image series
"1-"+str(numtemplate), # enter number of templates of doc file
str(int(xysearch))+","+str(int(xystep)), # translation search range, step size
str(int(firstring))+","+str(int(lastring)), # first and last ring for rotational correlation
"*", # initial angles
spyder.fileFilter(stackfile)+"@******", # unaligned image series
"1-"+str(numpart), # enter number of particles of doc file
"*", # initial angles
"0.0", # angle search
"1", # check mirrors and shift/rotate input
rundir+("/paramdoc%02d" % (iternum)), # output angles document file
)
mySpider.close()
"""
### convert spider rotation, shift data to python
docfile = rundir + ("/paramdoc%02d" % (iternum)) + dataext
picklefile = rundir + ("/paramdoc%02d" % (iternum)) + ".pickle"
if oldpartlist is not None and iternum > 1:
apDisplay.printMsg("updating particle doc info")
partlist = updateRefBasedDocFile(oldpartlist, docfile, picklefile)
elif iternum == 1:
apDisplay.printMsg("reading initial particle doc info")
partlist = readRefBasedDocFile(docfile, picklefile)
else:
apDisplay.printError(
"reading (not updating) particle doc info on iteration " +
str(iternum))
### write aligned stack -- with python loop
alignedstack = rundir + ("/alignedstack%02d" % (iternum))
alignStack(origstackfile, alignedstack, partlist, dataext)
### average stack
emancmd = ("proc2d " + alignedstack + dataext + " " + rundir +
("/avgimg%02d" % (iternum)) + ".mrc " + " average")
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
td1 = time.time() - t0
apDisplay.printMsg("completed alignment of " + str(numpart) +
" particles in " + apDisplay.timeString(td1))
if numpart < 1:
apDisplay.printError("Failed to find any particles")
return alignedstack + dataext, partlist
def start(self):
if self.params['method'] == 'any':
### startAny uses class averages
clusterstack, numimages = self.getClusterStack()
else:
### starticos, startoct, startcsym uses individual particles
clusterstack, numimages = self.getClusterParticles()
if self.params['method'] != 'any':
if self.params['numkeep'] is not None and numimages/10 < int(self.params['numkeep']):
apDisplay.printWarning("particle number of "+ self.params['numkeep']
+ " is greater than 10% of the number of selected classes")
elif self.params['numkeep'] is None:
self.params['numkeep'] = int(math.floor(numimages/20.0))+1
apDisplay.printWarning("numkeep was not defined, using %d particles"%(self.params['numkeep']))
nproc = apParam.getNumProcessors()
#construct command for each of the EMAN commonline method
if self.params['method'] == 'any':
startcmd = "startAny "+clusterstack+" proc="+str(nproc)
startcmd +=" sym="+self.symmdata['eman_name']
if self.params['mask'] is not None:
startcmd +=" mask="+str(self.params['mask'])
else:
maskrad = math.floor(self.clusterstackdata['clusterrun']['boxsize']/2.0)
startcmd +=" mask=%d"%(maskrad)
if self.params['rounds'] is not None:
startcmd +=" rounds="+str(self.params['rounds'])
elif self.params['method'] == 'csym':
startcmd = "startcsym "+clusterstack+" "
startcmd +=" "+str(self.params['numkeep'])
startcmd +=" sym="+self.symmdata['eman_name']
if self.params['imask'] is not None:
startcmd +=" imask="+self.params['imask']
elif self.params['method'] == 'oct':
startcmd = "startoct "+clusterstack+" "
startcmd +=" "+str(self.params['numkeep'])
elif self.params['method'] == 'icos':
startcmd = "starticos "+clusterstack+" "
startcmd +=" "+str(self.params['numkeep'])
if self.params['imask'] is not None:
startcmd +=" imask="+self.params['imask']
apDisplay.printMsg("Creating 3D model with EMAN function: start"+self.params['method'])
apFile.removeFile("threed.0a.mrc")
apFile.removeFile("eman.log")
apEMAN.executeEmanCmd(startcmd, verbose=False, logfile="eman.log")
#apEMAN.executeEmanCmd(startcmd, verbose=True)
finalmodelname = "threed-%s-eman_start%s.mrc"%(self.timestamp, self.params['method'])
finalmodelpath = os.path.join(self.params['rundir'], finalmodelname)
apDisplay.printMsg("Final model name: "+finalmodelname)
finalmodel = "threed.0a.mrc"
if os.path.isfile(finalmodel):
emancmd = "proc3d %s %s norm=0,1 origin=0,0,0"%(finalmodel, finalmodelpath)
#shutil.move(finalmodel, finalmodelpath)
apEMAN.executeEmanCmd(emancmd, verbose=True)
if not apVolume.isValidVolume(finalmodelpath):
apDisplay.printError("Created volume is not valid")
else:
apDisplay.printError("No 3d model was created")
### upload it
self.uploadDensity(finalmodelpath)
### chimera imaging
apChimera.renderSnapshots(finalmodelpath, contour=self.params['contour'],
zoom=self.params['zoom'], sym=self.symmdata['eman_name'])
apChimera.renderAnimation(finalmodelpath, contour=self.params['contour'],
zoom=self.params['zoom'], sym=self.symmdata['eman_name'])
### remove stack
if apFile.stackSize(clusterstack)/1024**2 > 10:
### file bigger than 10MB
apFile.removeStack(clusterstack)
apFile.removeFile("threed.0a.mrc")
def checkConflicts(self):
### first get all stack data
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### modify boxsize and pixelsize according to binning
self.boxsize = self.stack['boxsize']
self.clipsize = int(
math.floor(self.boxsize / float(self.params['bin'] * 2))) * 2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d" %
(self.params['clipsize'], self.clipsize))
self.clipsize = self.params['clipsize']
self.apix = self.stack['apix']
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
self.boxsize = clipsize
self.apix = self.apix * self.params['bin']
### basic error checking
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['ncls'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 1000000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles " +
str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(stackfile)))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > 5000:
apDisplay.printWarning(
"initial model calculation may not work with less than 5000 particles"
)
self.mpirun = self.checkMPI()
# if self.mpirun is None:
# apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
### SIMPLE defaults and error checking
if self.params['ring2'] is None:
self.params['ring2'] = (self.boxsize / 2) - 2
if self.params['ncls'] > 2000:
apDisplay.printError(
"number of classes should be less than 2000 for subsequent ORIGAMI run to work"
)
if self.params['ncls'] > self.params['numpart']:
self.params['ncls'] = self.params['numpart'] / self.params['minp']
if self.params['mask'] is None:
self.params['mask'] = (self.boxsize / 2) - 2
if self.params['mw'] is None:
apDisplay.printError(
"please specify the molecular weight (in kDa)")
def refBasedAlignParticles(stackfile, templatestack,
origstackfile,
xysearch, xystep,
numpart, numtemplate,
firstring=2, lastring=100,
dataext=".spi",
iternum=1, oldpartlist=None):
"""
inputs:
stack
template
search params
outputs:
aligned stack
rotation/shift params
"""
### setup
if dataext in templatestack:
templatestack = templatestack[:-4]
if dataext in stackfile:
stackfile = stackfile[:-4]
if dataext in origstackfile:
origstackfile = origstackfile[:-4]
t0 = time.time()
rundir = "alignments"
apParam.createDirectory(rundir)
nproc = apParam.getNumProcessors()
### remove previous iterations
apFile.removeFile(rundir+"/paramdoc%02d%s" % (iternum, dataext))
### perform alignment, should I use 'AP SH' instead?
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("AP MQ",
spyder.fileFilter(templatestack)+"@**", # reference image series
"1-"+str(numtemplate), # enter number of templates of doc file
str(int(xysearch))+","+str(int(xystep)), # translation search range, step size
str(int(firstring))+","+str(int(lastring)), # first and last ring for rotational correlation
spyder.fileFilter(stackfile)+"@******", # unaligned image series
"1-"+str(numpart), # enter number of particles of doc file
rundir+("/paramdoc%02d" % (iternum)), # output angles document file
)
mySpider.close()
"""
### perform alignment, should I use 'AP SH' instead?
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("AP SH",
spyder.fileFilter(templatestack)+"@**", # reference image series
"1-"+str(numtemplate), # enter number of templates of doc file
str(int(xysearch))+","+str(int(xystep)), # translation search range, step size
str(int(firstring))+","+str(int(lastring)), # first and last ring for rotational correlation
"*", # initial angles
spyder.fileFilter(stackfile)+"@******", # unaligned image series
"1-"+str(numpart), # enter number of particles of doc file
"*", # initial angles
"0.0", # angle search
"1", # check mirrors and shift/rotate input
rundir+("/paramdoc%02d" % (iternum)), # output angles document file
)
mySpider.close()
"""
### convert spider rotation, shift data to python
docfile = rundir+("/paramdoc%02d" % (iternum))+dataext
picklefile = rundir+("/paramdoc%02d" % (iternum))+".pickle"
if oldpartlist is not None and iternum > 1:
apDisplay.printMsg("updating particle doc info")
partlist = updateRefBasedDocFile(oldpartlist, docfile, picklefile)
elif iternum == 1:
apDisplay.printMsg("reading initial particle doc info")
partlist = readRefBasedDocFile(docfile, picklefile)
else:
apDisplay.printError("reading (not updating) particle doc info on iteration "+str(iternum))
### write aligned stack -- with python loop
alignedstack = rundir+("/alignedstack%02d" % (iternum))
alignStack(origstackfile, alignedstack, partlist, dataext)
### average stack
emancmd = ( "proc2d "+alignedstack+dataext+" "
+rundir+("/avgimg%02d" % (iternum))+".mrc "
+" average")
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
td1 = time.time()-t0
apDisplay.printMsg("completed alignment of "+str(numpart)
+" particles in "+apDisplay.timeString(td1))
if numpart < 1:
apDisplay.printError("Failed to find any particles")
return alignedstack+dataext, partlist
