def checkConflicts(self):
if self.params["stackid"] is None:
apDisplay.printError("stack id was not defined")
if self.params["modelid"] is None:
apDisplay.printError("model id was not defined")
if self.params["runname"] is None:
apDisplay.printError("missing a reconstruction name")
if self.params["last"] is None:
self.params["last"] = apStack.getNumberStackParticlesFromId(self.params["stackid"])
self.boxsize = apStack.getStackBoxsize(self.params["stackid"], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
if self.params["reconstackid"] is not None:
reconboxsize = apStack.getStackBoxsize(self.params["reconstackid"], msg=False)
reconapix = apStack.getStackPixelSizeFromStackId(self.params["reconstackid"])
refinenumpart = apStack.getNumberStackParticlesFromId(self.params["stackid"])
reconnumpart = apStack.getNumberStackParticlesFromId(self.params["reconstackid"])
if reconboxsize != self.boxsize:
apDisplay.printError("Boxsize do not match for stacks")
if reconapix != self.apix:
apDisplay.printError("Pixelsize do not match for stacks")
if refinenumpart != reconnumpart:
apDisplay.printError("Number of particles do not match for stacks")
maxmask = math.floor(self.apix * (self.boxsize - 10) / 2.0)
if self.params["mask"] is None:
apDisplay.printWarning("mask was not defined, setting to boxsize: %d" % (maxmask))
self.params["mask"] = maxmask
if self.params["mask"] > maxmask:
apDisplay.printWarning("mask was too big, setting to boxsize: %d" % (maxmask))
self.params["mask"] = maxmask
if self.params["noctf"] is True:
apDisplay.printWarning("Using no CTF method")
self.params["wgh"] = -1.0
if self.params["nodes"] > 1 and self.params["cluster"] is False:
apDisplay.printError("cluster mode must be enabled to run on more than 1 node")
if self.params["ppn"] is None:
if self.params["cluster"] is True:
apDisplay.printError("you must define ppn for cluster mode")
self.params["ppn"] = apParam.getNumProcessors()
apDisplay.printMsg("Setting number of processors to %d" % (self.params["ppn"]))
if self.params["rpn"] is None:
self.params["rpn"] = self.params["ppn"]
self.params["nproc"] = self.params["nodes"] * self.params["rpn"]
### get the symmetry data
if self.params["sym"] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params["sym"])
self.params["symm_id"] = self.symmdata.dbid
self.params["symm_name"] = self.symmdata["eman_name"]
apDisplay.printMsg("Selected symmetry %s with id %s" % (self.symmdata["eman_name"], self.symmdata.dbid))
### set cs value
self.params["cs"] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
### get stack data
self.stack = {}
if self.params['stackid'] is not None:
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
else:
self.stack['data'] = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
self.stack['apix'] = self.stack['pixelsize']
self.stack['boxsize'] = self.stack['boxsize']
self.stack['file'] = self.stack['imagicfile']
### check conflicts
if self.params['stackid'] is None and self.params['alignstackid'] is None:
apDisplay.printError("stack id OR alignstack id was not defined")
if self.params['stackid'] is not None and self.params['alignstackid'] is not None:
apDisplay.printError("either specify stack id OR alignstack id, not both")
if self.params['generations'] is None:
apDisplay.printError("number of generations was not provided")
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
if self.params['numpart'] > apFile.numImagesInStack(self.stack['file']):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(self.stack['file'])))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(self.stack['file'])
boxsize = apStack.getStackBoxsize(self.params['stackid'])
if self.params['ou'] is None:
self.params['ou'] = (boxsize / 2.0) - 2
self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
self.mpirun = self.checkMPI()
if self.mpirun is None:
apDisplay.printError("There is no MPI installed")
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['runname'] is None:
apDisplay.printError("missing a reconstruction name")
if self.params['last'] is None:
self.params['last'] = apStack.getNumberStackParticlesFromId(self.params['stackid'])
self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
if self.params['reconstackid'] is not None:
reconboxsize = apStack.getStackBoxsize(self.params['reconstackid'], msg=False)
reconapix = apStack.getStackPixelSizeFromStackId(self.params['reconstackid'])
refinenumpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
reconnumpart = apStack.getNumberStackParticlesFromId(self.params['reconstackid'])
if reconboxsize != self.boxsize:
apDisplay.printError("Boxsize do not match for stacks")
if reconapix != self.apix:
apDisplay.printError("Pixelsize do not match for stacks")
if refinenumpart != reconnumpart:
apDisplay.printError("Number of particles do not match for stacks")
maxmask = math.floor(self.apix*(self.boxsize-10)/2.0)
if self.params['mask'] is None:
apDisplay.printWarning("mask was not defined, setting to boxsize: %d"%(maxmask))
self.params['mask'] = maxmask
if self.params['mask'] > maxmask:
apDisplay.printWarning("mask was too big, setting to boxsize: %d"%(maxmask))
self.params['mask'] = maxmask
if self.params['noctf'] is True:
apDisplay.printWarning("Using no CTF method")
self.params['wgh'] = -1.0
if self.params['nodes'] > 1 and self.params['cluster'] is False:
apDisplay.printError("cluster mode must be enabled to run on more than 1 node")
if self.params['ppn'] is None:
if self.params['cluster'] is True:
apDisplay.printError("you must define ppn for cluster mode")
self.params['ppn'] = apParam.getNumProcessors()
apDisplay.printMsg("Setting number of processors to %d"%(self.params['ppn']))
if self.params['rpn'] is None:
self.params['rpn'] = self.params['ppn']
self.params['nproc'] = self.params['nodes']*self.params['rpn']
### get the symmetry data
if self.params['sym'] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params['sym'])
self.params['symm_id'] = self.symmdata.dbid
self.params['symm_name'] = self.symmdata['eman_name']
apDisplay.printMsg("Selected symmetry %s with id %s"%(self.symmdata['eman_name'], self.symmdata.dbid))
### set cs value
self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['start'] is None:
apDisplay.printError("a number of starting classes was not provided")
if self.params['end'] is None:
apDisplay.printError("a number of ending classes was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
# check for virtual stack
self.params['virtualdata'] = None
if not os.path.isfile(stackfile):
vstackdata = apStack.getVirtualStackParticlesFromId(self.params['stackid'])
npart = len(vstackdata['particles'])
self.params['virtualdata'] = vstackdata
else:
npart = apFile.numImagesInStack(stackfile)
if self.params['numpart'] is None:
self.params['numpart'] = npart
elif self.params['numpart'] > npart:
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.workingboxsize = math.floor(self.boxsize/self.params['bin'])
if not self.params['mask']:
self.params['mask'] = (self.boxsize/2)-2
self.workingmask = math.floor(self.params['mask']/self.params['bin'])
if self.params['mramethod'] == 'imagic':
self.imagicroot = apIMAGIC.checkImagicExecutablePath()
self.imagicversion = apIMAGIC.getImagicVersion(self.imagicroot)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['rad'] is None:
apDisplay.printError("no particle radius set")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['lastring'] is None:
apDisplay.printError("a last ring radius was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if self.params['incr'] is None:
apDisplay.printError("angular increments are not specified")
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
numstackp = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > numstackp:
apDisplay.printError("trying to use more particles "+str(numstackp)
+" than available "+str(numstackp))
elif self.params['numpart'] == 0:
apDisplay.printWarning("using all "+str(numstackp)+" particles")
self.params['numpart'] = numstackp
boxsize = apStack.getStackBoxsize(self.params['stackid'])
if self.params['lastring'] > boxsize/2-2:
apDisplay.printError("last ring radius is too big for boxsize "
+str(self.params['lastring'])+" > "+str(boxsize/2-2))
if self.params['lastring']+self.params['xysearch'] > boxsize/2-2:
apDisplay.printError("last ring plus xysearch radius is too big for boxsize "
+str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2))
if (self.params['xysearch'] % self.params['xystep']) > 0:
apDisplay.printError("translational search (xy-search) must be divisible by search step (xy-step)")
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['description'] is None:
apDisplay.printError("run description was not defined")
if self.params['templatelist'] is None:
apDisplay.printError("template list was not provided")
if self.params['lastring'] is None:
apDisplay.printError("a last ring radius was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])/self.params['bin']
if self.params['lastring'] > boxsize/2-2:
apDisplay.printError("last ring radius is too big for boxsize "
+str(self.params['lastring'])+" > "+str(boxsize/2-2))
if self.params['lastring']+self.params['xysearch'] > boxsize/2-2:
apDisplay.printError("last ring plus xysearch radius is too big for boxsize "
+str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2))
### convert / check template data
self.templatelist = self.params['templatelist'].strip().split(",")
if not self.templatelist or type(self.templatelist) != type([]):
apDisplay.printError("could not parse template list="+self.params['templatelist'])
self.params['numtemplate'] = len(self.templatelist)
apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates")
def setupParticles(self):
self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed")
self.stack = {}
self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"])
self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"])
self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"])
self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"])
boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2
apix = self.stack["apix"] * self.params["bin"]
proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"])
if self.params["bin"] > 1:
clipsize = boxsize * self.params["bin"]
proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize)
if self.params["highpass"] > 1:
proccmd += " hp=" + str(self.params["highpass"])
if self.params["lowpass"] > 1:
proccmd += " lp=" + str(self.params["lowpass"])
proccmd += " last=" + str(self.params["numpart"] - 1)
apFile.removeStack(self.params["localstack"], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"])
return (boxsize, apix)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['numrefs'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
self.mpirun = self.checkMPI()
if self.mpirun is None:
apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
if self.params['nproc'] < 2:
apDisplay.printError("Only the MPI version of CL2D is currently supported, must run with > 1 CPU")
def start(self):
"""
This is the core of your function.
"""
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp+".hed")
apFile.removeStack(filtered_stackfile, warn=False)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
boxsize = apStack.getStackBoxsize(self.params['stackid'])
emancmd = "proc2d %s %s apix=%.3f "%(original_stackfile, filtered_stackfile, apix)
if self.params['lowpass'] is not None:
emancmd += " lp=%.3f "%(self.params['lowpass'])
if self.params['highpass'] is not None:
emancmd += " hp=%.3f "%(self.params['highpass'])
if self.params['bin'] is not None and self.params['bin'] > 1:
## determine a multiple of the bin that is divisible by 2 and less than the boxsize
clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2*self.params['bin']
emancmd += " shrink=%d clip=%d,%d "%(self.params['bin'], clipsize, clipsize)
emancmd += " last=%d "%(self.params['numpart']-1)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
### confirm that it worked
if self.params['numpart'] != apFile.numImagesInStack(filtered_stackfile):
apDisplay.printError("Missing particles in stack")
### run the radon transform code
self.radonAlign(filtered_stackfile)
### insert info into database
self.commitToDatabase()
def getBoxSize(self):
boxsize = apStack.getStackBoxsize(self.params['tiltstackid'])
if self.params['tiltbin'] == 1:
return boxsize
newbox = int(
math.floor(boxsize / float(self.params['tiltbin']) / 2.0) * 2.0)
return newbox
def setupParticles(self):
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
apix = self.stack['apix']*self.params['bin']
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1:
clipsize = boxsize*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
if self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
proccmd += " last="+str(self.params['numpart']-1)
apFile.removeStack(self.params['localstack'], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
return (boxsize, apix)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
#if self.params['description'] is None:
# apDisplay.printError("run description was not defined")
if self.params['numrefs'] is None:
apDisplay.printError("a number of classes was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if not self.params['fastmode'] in self.fastmodes:
apDisplay.printError("fast mode must be on of: "+str(self.fastmodes))
if not self.params['converge'] in self.convergemodes:
apDisplay.printError("converge mode must be on of: "+str(self.convergemodes))
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.checkNoRefRun()
### convert stack to spider
spiderstack = self.createSpiderFile()
### create initialization template
if self.params['initmethod'] == 'allaverage':
templatefile = self.averageTemplate()
elif self.params['initmethod'] == 'selectrand':
templatefile = self.selectRandomParticles()
elif self.params['initmethod'] == 'randpart':
templatefile = self.pickRandomParticle()
elif self.params['initmethod'] == 'template':
templatefile = self.getTemplate()
else:
apDisplay.printError("unknown initialization method defined: "
+str(self.params['initmethod'])+" not in "+str(self.initmethods))
apDisplay.printColor("Running spider this can take awhile","cyan")
### run the alignment
aligntime = time.time()
pixrad = int(round(self.params['partrad']/self.stack['apix']/self.params['bin']))
alignedstack, self.partlist = alignment.refFreeAlignParticles(
spiderstack, templatefile,
self.params['numpart'], pixrad,
self.params['firstring'], self.params['lastring'],
rundir = ".")
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=False)
### convert stack to imagic
imagicstack = self.convertSpiderStack(alignedstack)
apFile.removeFile(alignedstack)
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = aligntime
self.insertNoRefRun(imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def checkConflicts(self):
if self.params['runname'] is None:
apDisplay.printError("enter a stack run name, e.g. combinestack1")
if self.params['description'] is None:
apDisplay.printError("enter a stack description")
if self.params['stacks'] and ',' in self.params['stacks']:
#remember stackids are a list of strings
stackids = self.params['stacks'].split(',')
self.params['stackids'] = stackids
else:
apDisplay.printError("enter a list of stack ids to combine, e.g. --stackids=11,14,7")
### check to make sure all pixel and box size are the same
self.newboxsize = None
self.newpixelsize = None
for stackidstr in self.params['stackids']:
if not re.match("^[0-9]+$", stackidstr):
apDisplay.printError("Stack id '%s' is not an integer"%(stackidstr))
stackid = int(stackidstr)
boxsize = apStack.getStackBoxsize(stackid, msg=False)
pixelsize = apStack.getStackPixelSizeFromStackId(stackid, msg=False)
apDisplay.printMsg("Stack id: %d\tBoxsize: %d\tPixelsize: %.3f"%(stackid, boxsize, pixelsize))
if self.newboxsize is None:
self.newboxsize = boxsize
if self.newpixelsize is None:
self.newpixelsize = pixelsize
if boxsize != self.newboxsize:
apDisplay.printError("Trying to combine stacks with different box sizes")
if abs(pixelsize - self.newpixelsize) > 0.01:
apDisplay.printError("Trying to combine stacks with different pixel sizes")
def start(self):
"""
this is the main component of the script
where all the processing is done
"""
### initialize some variables
self.runq = None
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
apDisplay.printMsg("Pixel size: %.5f"%(self.apix))
self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
apDisplay.printMsg("Box size: %d"%(self.boxsize))
self.checkResults()
self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
self.numiter = self.getNumberOfIterations()
for i in range(self.numiter):
iternum = i+1
apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green")
self.uploadIteration(iternum)
reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir'])
if reconrunid:
apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid'])
apRecon.setGoodBadParticlesFromReconId(reconrunid)
else:
apDisplay.printWarning("Could not find recon run id")
def start(self):
"""
this is the main component of the script
where all the processing is done
"""
### initialize some variables
self.runq = None
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
apDisplay.printMsg("Pixel size: %.5f"%(self.apix))
self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
apDisplay.printMsg("Box size: %d"%(self.boxsize))
self.checkResults()
self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
self.numiter = self.getNumberOfIterations()
for i in range(self.numiter):
iternum = i+1
apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green")
self.uploadIteration(iternum)
reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir'])
if reconrunid:
apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid'])
apRecon.setGoodBadParticlesFromReconId(reconrunid)
else:
apDisplay.printWarning("Could not find recon run id")
def setupParticles(self):
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
apix = self.stack['apix']*self.params['bin']
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1:
clipsize = boxsize*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
if self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
proccmd += " last="+str(self.params['numpart']-1)
apFile.removeStack(self.params['localstack'], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
return (boxsize, apix)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['description'] is None:
apDisplay.printError("run description was not defined")
if self.params['templatelist'] is None:
apDisplay.printError("template list was not provided")
if self.params['lastring'] is None:
apDisplay.printError("a last ring radius was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])/self.params['bin']
if self.params['lastring'] > boxsize/2-2:
apDisplay.printError("last ring radius is too big for boxsize "
+str(self.params['lastring'])+" > "+str(boxsize/2-2))
if self.params['lastring']+self.params['xysearch'] > boxsize/2-2:
apDisplay.printError("last ring plus xysearch radius is too big for boxsize "
+str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2))
### convert / check template data
self.templatelist = self.params['templatelist'].strip().split(",")
if not self.templatelist or type(self.templatelist) != type([]):
apDisplay.printError("could not parse template list="+self.params['templatelist'])
self.params['numtemplate'] = len(self.templatelist)
apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates")
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("Please provide a user id, e.g. --stackid=15")
if not 'radonshift' in dir(radermacher):
apDisplay.printError("Radermacher python module is too old, please update")
boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
pixelsize = apStack.getStackPixelSizeFromStackId(self.params['stackid'], msg=False)
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'], msg=False)
apDisplay.printMsg("\n\n")
### get info about the stack
apDisplay.printMsg("Information about stack id %d"%(self.params['stackid']))
apDisplay.printMsg("\tboxsize %d pixels"%(boxsize))
apDisplay.printMsg("\tpixelsize %.3f Angstroms"%(pixelsize))
apDisplay.printMsg("\tsize %d particles"%(numpart))
if self.params['maskrad'] is None:
self.params['pixelmaskrad'] = boxsize/2
self.params['maskrad'] = self.params['pixelmaskrad']*pixelsize
else:
self.params['pixelmaskrad'] = self.params['maskrad']/pixelsize
if self.params['pixelmaskrad'] > boxsize/2:
apDisplay.printError("Mask radius larger than boxsize")
if self.params['numpart'] is None:
self.params['numpart'] = numpart
if self.params['numpart'] > numpart:
apDisplay.printError("Requested more particles than available in stack")
if self.params['numrefs'] is None:
### num refs should go approx. as the sqrt of the number of particles
self.params['numrefs'] = int(math.ceil(math.sqrt(self.params['numpart'])))+1
def checkConflicts(self):
if self.params["stackid"] is None:
apDisplay.printError("Stack id was not defined")
if self.params["mandir"] is None:
apDisplay.printError("Directory containing mandatory files was not defined")
if self.params["description"] is None:
apDisplay.printError("Run description was not defined")
stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False)
stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"])
if self.params["numpart"] is None:
self.params["numpart"] = apFile.numImagesInStack(stackfile)
apDisplay.printWarning("Number of particles not defined, processing full stack")
if self.params["numpart"] > apFile.numImagesInStack(stackfile):
apDisplay.printError(
"trying to use more particles "
+ str(self.params["numpart"])
+ " than available "
+ str(apFile.numImagesInStack(stackfile))
)
if self.params["replen"] is None:
apDisplay.printError("Helical repeat was not defined")
if self.params["subunits"] is None:
apDisplay.printError("Number of subunits was not defined")
if self.params["diameter"] is None:
apDisplay.printError("Filament diameter was not defined")
if self.params["diaminner"] is None:
apDisplay.printError("Inner filament diameter was not defined")
if self.params["rescut"] is None:
apDisplay.printError("Phase residual cutoff was not defined")
if self.params["filval"] is None:
apDisplay.printError("Filter value was not defined")
if self.params["bin"] is None:
self.params["bin"] = 1
apDisplay.printWarning("Binning factor was not defined, binning by one")
if self.params["cont"] is None:
apDisplay.printError("Must specify if a contrast change is needed")
if self.params["prehip"] is None:
apDisplay.printError("Must specify if prehip needs to be executed or not")
step = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
self.params["step"] = step
onerep = math.floor(self.params["replen"] / step)
calcyht2 = math.floor(self.params["diameter"] / step)
boxsize = apStack.getStackBoxsize(self.params["stackid"])
self.params["boxsize"] = boxsize
if self.params["xlngth"] < onerep:
apDisplay.printError("Filament segment length can not be less than one helical repeat")
if self.params["xlngth"] > boxsize:
apDisplay.printError("Filament segment length can not be greater than stack box size")
if self.params["yht2"] < calcyht2:
apDisplay.printError("Filament box height can not be less than filament diameter")
if self.params["padval"] < self.params["xlngth"]:
apDisplay.printError("Pad value can not be less than the filament segment length")
if not self.checkval(self.params["yht2"]):
apDisplay.printError("Filament box height must be a power of two")
if not self.checkval(self.params["padval"]):
apDisplay.printError("Pad value must be a power of two")
def __initializeStack(self):
# This sets parameters of our new stack based on the params of the original stack
# TODO: use originalStackData to populate these
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.stack['format'] = 'imagic'
self.stack['phaseflipped'] = self.originalStackData.phaseFlipped
def __initializeStack(self):
# This sets parameters of our new stack based on the params of the original stack
# TODO: use originalStackData to populate these
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.stack['format'] = 'imagic'
self.stack['phaseflipped'] = self.originalStackData.phaseFlipped
def convertVolToSpider(self,
mrcvolfile=None,
modelid=None,
apix=None,
spivolfile=None):
"""
takes the mrc volume and creates a spider file ready for processing
"""
if modelid is not None:
initModelData = appiondata.ApInitialModelData.direct_query(modelid)
mrcvolfile = initModelData['path']['path'] + "/" + initModelData[
'name']
apix = initModelData['pixelsize']
stackapix = apStack.getStackPixelSizeFromStackId(
self.params['partstackid']) * self.params['bin']
stackboxsize = apStack.getStackBoxsize(
self.params['partstackid']) / self.params['bin']
print apix
print mrcvolfile
if not os.path.isfile(mrcvolfile):
apDisplay.printError("volfile does not exist: " + mrcvolfile)
### first high pass filter particles
#apDisplay.printMsg("pre-filtering particles")
#apix = apStack.getStackPixelSizeFromStackId(self.params['partstackid'])
#emancmd = ("proc3d "+mrcvolfile+" "+emanstackfile
# +" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
# +" inplace")
#apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc3d "
emancmd += mrcvolfile + " "
if spivolfile is None:
spivolfile = os.path.join(self.params['rundir'], "threed-0a.spi")
apFile.removeFile(spivolfile, warn=True)
emancmd += spivolfile + " "
emancmd += "scale=" + str(apix / stackapix) + " "
emancmd += "clip=" + str(stackboxsize) + "," + str(
stackboxsize) + "," + str(stackboxsize) + " "
emancmd += "spidersingle"
starttime = time.time()
apDisplay.printColor("Running spider volume conversion", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor(
"finished conversion in " +
apDisplay.timeString(time.time() - starttime), "cyan")
return spivolfile
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("Stack id was not defined")
if self.params['mandir'] is None:
apDisplay.printError("Directory containing mandatory files was not defined")
if self.params['description'] is None:
apDisplay.printError("Run description was not defined")
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
apDisplay.printWarning("Number of particles not defined, processing full stack")
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])+" than available "+str(apFile.numImagesInStack(stackfile)))
if self.params['replen'] is None:
apDisplay.printError("Helical repeat was not defined")
if self.params['subunits'] is None:
apDisplay.printError("Number of subunits was not defined")
if self.params['diameter'] is None:
apDisplay.printError("Filament diameter was not defined")
if self.params['diaminner'] is None:
apDisplay.printError("Inner filament diameter was not defined")
if self.params['rescut'] is None:
apDisplay.printError("Phase residual cutoff was not defined")
if self.params['filval'] is None:
apDisplay.printError("Filter value was not defined")
if self.params['bin'] is None:
self.params['bin'] = 1
apDisplay.printWarning("Binning factor was not defined, binning by one")
if self.params['cont'] is None:
apDisplay.printError("Must specify if a contrast change is needed")
if self.params['prehip'] is None:
apDisplay.printError("Must specify if prehip needs to be executed or not")
step = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.params['step'] = step
onerep = math.floor(self.params['replen'] / step)
calcyht2 = math.floor(self.params['diameter'] / step)
boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.params['boxsize'] = boxsize
if self.params['xlngth'] < onerep:
apDisplay.printError("Filament segment length can not be less than one helical repeat")
if self.params['xlngth'] > boxsize:
apDisplay.printError("Filament segment length can not be greater than stack box size")
if self.params['yht2'] < calcyht2:
apDisplay.printError("Filament box height can not be less than filament diameter")
if self.params['padval'] < self.params['xlngth']:
apDisplay.printError("Pad value can not be less than the filament segment length")
if not self.checkval(self.params['yht2']):
apDisplay.printError("Filament box height must be a power of two")
if not self.checkval(self.params['padval']):
apDisplay.printError("Pad value must be a power of two")
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['numrefs'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
self.stackdata = apStack.getOnlyStackData(self.params['stackid'],
msg=False)
stackfile = os.path.join(self.stackdata['path']['path'],
self.stackdata['name'])
# check for virtual stack
self.params['virtualdata'] = None
if not os.path.isfile(stackfile):
vstackdata = apStack.getVirtualStackParticlesFromId(
self.params['stackid'])
npart = len(vstackdata['particles'])
self.params['virtualdata'] = vstackdata
else:
npart = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > npart:
apDisplay.printError("trying to use more particles " +
str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.clipsize = int(math.floor(
boxsize / float(self.params['bin'] * 2))) * 2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d" %
(self.params['clipsize'], self.clipsize))
self.clipsize = self.params['clipsize']
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
self.mpirun = self.checkMPI()
if self.mpirun is None:
apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = self.params['nodes'] * self.params['ppn']
if self.params['nproc'] < 2:
apDisplay.printError(
"Only the MPI version of CL2D is currently supported, must run with > 1 CPU"
)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['description'] is None:
apDisplay.printError("run description was not defined")
### convert / check template data
if self.params['templatelist'] is None:
apDisplay.printError("template list was not provided")
self.templatelist = self.params['templatelist'].strip().split(",")
if not self.templatelist or type(self.templatelist) != type([]):
apDisplay.printError("could not parse template list=" +
self.params['templatelist'])
self.params['numtemplate'] = len(self.templatelist)
apDisplay.printMsg("Found " + str(self.params['numtemplate']) +
" templates")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if not self.params['fastmode'] in self.fastmodes:
apDisplay.printError("fast mode must be on of: " +
str(self.fastmodes))
if not self.params['converge'] in self.convergemodes:
apDisplay.printError("converge mode must be on of: " +
str(self.convergemodes))
maxparticles = 150000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles " +
str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.clipsize = int(math.floor(
boxsize / float(self.params['bin'] * 2))) * 2
if self.params['clipsize'] is not None:
if self.params['clipsize'] >= self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d" %
(self.params['clipsize'], self.clipsize))
self.clipsize = self.params['clipsize']
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['runname'] is None:
apDisplay.printError("new runname was not defined")
self.params['totalpart'] = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if 'last' not in self.params.keys() or self.params['last'] is None:
self.params['last'] = self.params['totalpart']
self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.refineboxsize = self.boxsize * self.params['bin']
### set cs value
self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
self.modelids = map((lambda x: int(x)),self.params['modelid'].split(','))
self.checkPackageConflicts()
def convertVolToSpider(self, mrcvolfile=None, modelid=None, apix=None, spivolfile=None):
"""
takes the mrc volume and creates a spider file ready for processing
"""
if modelid is not None:
initModelData = appiondata.ApInitialModelData.direct_query(modelid)
mrcvolfile = initModelData["path"]["path"] + "/" + initModelData["name"]
apix = initModelData["pixelsize"]
stackapix = apStack.getStackPixelSizeFromStackId(self.params["partstackid"]) * self.params["bin"]
stackboxsize = apStack.getStackBoxsize(self.params["partstackid"]) / self.params["bin"]
print apix
print mrcvolfile
if not os.path.isfile(mrcvolfile):
apDisplay.printError("volfile does not exist: " + mrcvolfile)
### first high pass filter particles
# apDisplay.printMsg("pre-filtering particles")
# apix = apStack.getStackPixelSizeFromStackId(self.params['partstackid'])
# emancmd = ("proc3d "+mrcvolfile+" "+emanstackfile
# +" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
# +" inplace")
# apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc3d "
emancmd += mrcvolfile + " "
if spivolfile is None:
spivolfile = os.path.join(self.params["rundir"], "threed-0a.spi")
apFile.removeFile(spivolfile, warn=True)
emancmd += spivolfile + " "
emancmd += "scale=" + str(apix / stackapix) + " "
emancmd += "clip=" + str(stackboxsize) + "," + str(stackboxsize) + "," + str(stackboxsize) + " "
emancmd += "spidersingle"
starttime = time.time()
apDisplay.printColor("Running spider volume conversion", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished conversion in " + apDisplay.timeString(time.time() - starttime), "cyan")
return spivolfile
def createFastFreeHandInputLineTemplate(self):
self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.apix = 6.02
one_particledata = apStack.getOneParticleFromStackId(self.params['stackid'], particlenumber=1, msg=True)
scopedata = one_particledata['particle']['image']['scope']
ht_kv = scopedata['high tension'] / 1000.0
cs_mm = scopedata['tem']['cs'] * 1000
angSearch = self.params['angSearch']
constant_inputlines = [
'%.3f,%.3f,%2f,%.f' %(self.apix,self.params['snr'],cs_mm,ht_kv)
,'1,0'
,'../%s' %(self.stackfile)
,'../%s' %(self.modelfile)
,'../%s' %(self.frealign_paramfile)
,'out.mrc'
,'-%d,%d,-%d,%d' %(angSearch,angSearch,angSearch,angSearch)
,'%.1f,%.2f,%.2f' %(self.params['minres'],self.params['maxres'],self.params['radius'])
,'%s' %(self.params['scoringtype'].upper())
]
return constant_inputlines
def createFastFreeHandInputLineTemplate(self):
self.boxsize = apStack.getStackBoxsize(self.params["stackid"], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
self.apix = 6.02
one_particledata = apStack.getOneParticleFromStackId(self.params["stackid"], particlenumber=1, msg=True)
scopedata = one_particledata["particle"]["image"]["scope"]
ht_kv = scopedata["high tension"] / 1000.0
cs_mm = scopedata["tem"]["cs"] * 1000
angSearch = self.params["angSearch"]
constant_inputlines = [
"%.3f,%.3f,%2f,%.f" % (self.apix, self.params["snr"], cs_mm, ht_kv),
"1,0",
"../%s" % (self.stackfile),
"../%s" % (self.modelfile),
"../%s" % (self.frealign_paramfile),
"out.mrc",
"-%d,%d,-%d,%d" % (angSearch, angSearch, angSearch, angSearch),
"%.1f,%.2f,%.2f" % (self.params["minres"], self.params["maxres"], self.params["radius"]),
"%s" % (self.params["scoringtype"].upper()),
]
return constant_inputlines
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['description'] is None:
apDisplay.printError("run description was not defined")
### convert / check template data
if self.params['templatelist'] is None:
apDisplay.printError("template list was not provided")
self.templatelist = self.params['templatelist'].strip().split(",")
if not self.templatelist or type(self.templatelist) != type([]):
apDisplay.printError("could not parse template list="+self.params['templatelist'])
self.params['numtemplate'] = len(self.templatelist)
apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if not self.params['fastmode'] in self.fastmodes:
apDisplay.printError("fast mode must be on of: "+str(self.fastmodes))
if not self.params['converge'] in self.convergemodes:
apDisplay.printError("converge mode must be on of: "+str(self.convergemodes))
maxparticles = 150000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] >= self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['runname'] is None:
apDisplay.printError("new runname was not defined")
self.params['totalpart'] = apStack.getNumberStackParticlesFromId(
self.params['stackid'])
if 'last' not in self.params.keys() or self.params['last'] is None:
self.params['last'] = self.params['totalpart']
self.boxsize = apStack.getStackBoxsize(self.params['stackid'],
msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.refineboxsize = self.boxsize * self.params['bin']
### set cs value
self.params['cs'] = apInstrument.getCsValueFromSession(
self.getSessionData())
self.modelids = map((lambda x: int(x)),
self.params['modelid'].split(','))
self.checkPackageConflicts()
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['rad'] is None:
apDisplay.printError("no particle radius set")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['lastring'] is None:
apDisplay.printError("a last ring radius was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if self.params['incr'] is None:
apDisplay.printError("angular increments are not specified")
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
numstackp = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > numstackp:
apDisplay.printError("trying to use more particles " +
str(numstackp) + " than available " +
str(numstackp))
elif self.params['numpart'] == 0:
apDisplay.printWarning("using all " + str(numstackp) +
" particles")
self.params['numpart'] = numstackp
boxsize = apStack.getStackBoxsize(self.params['stackid'])
if self.params['lastring'] > boxsize / 2 - 2:
apDisplay.printError("last ring radius is too big for boxsize " +
str(self.params['lastring']) + " > " +
str(boxsize / 2 - 2))
if self.params['lastring'] + self.params['xysearch'] > boxsize / 2 - 2:
apDisplay.printError(
"last ring plus xysearch radius is too big for boxsize " +
str(self.params['lastring'] + self.params['xysearch']) +
" > " + str(boxsize / 2 - 2))
if (self.params['xysearch'] % self.params['xystep']) > 0:
apDisplay.printError(
"translational search (xy-search) must be divisible by search step (xy-step)"
)
def checkConflicts(self):
if self.params['runname'] is None:
apDisplay.printError("enter a stack run name, e.g. combinestack1")
if self.params['description'] is None:
apDisplay.printError("enter a stack description")
if self.params['stacks'] and ',' in self.params['stacks']:
#remember stackids are a list of strings
stackids = self.params['stacks'].split(',')
self.params['stackids'] = stackids
else:
apDisplay.printError(
"enter a list of stack ids to combine, e.g. --stackids=11,14,7"
)
### check to make sure all pixel and box size are the same
self.newboxsize = None
self.newpixelsize = None
for stackidstr in self.params['stackids']:
if not re.match("^[0-9]+$", stackidstr):
apDisplay.printError("Stack id '%s' is not an integer" %
(stackidstr))
stackid = int(stackidstr)
boxsize = apStack.getStackBoxsize(stackid, msg=False)
pixelsize = apStack.getStackPixelSizeFromStackId(stackid,
msg=False)
apDisplay.printMsg("Stack id: %d\tBoxsize: %d\tPixelsize: %.3f" %
(stackid, boxsize, pixelsize))
if self.newboxsize is None:
self.newboxsize = boxsize
if self.newpixelsize is None:
self.newpixelsize = pixelsize
if boxsize != self.newboxsize:
apDisplay.printError(
"Trying to combine stacks with different box sizes")
if abs(pixelsize - self.newpixelsize) > 0.01:
apDisplay.printError(
"Trying to combine stacks with different pixel sizes")
def getBoxSize(self): boxsize = apStack.getStackBoxsize(self.params['tiltstackid']) if self.params['tiltbin'] == 1: return boxsize newbox = int( math.floor( boxsize / float(self.params['tiltbin']) / 2.0)* 2.0 ) return newbox
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
# symmetry info
if self.params['sym'] == 'Icos':
self.params['symtype'] = 'I'
self.params['symfold'] = None
else:
self.params['symtype'] = self.params['sym'][0]
self.params['symfold'] = int(self.params['sym'][1:])
# eman "d" symmetry is spider "ci"
if self.params['symtype'].upper() == "D":
self.params['symtype'] = "CI"
# create symmetry doc file
sydoc = "sym.spi"
refine.symmetryDoc(self.params['symtype'], self.params['symfold'],
sydoc)
# convert incr to array of increments
ang_inc = self.params['incr'].split(',')
self.params['numiter'] = len(ang_inc)
self.params['increments'] = []
for i in range(0, self.params['numiter']):
self.params['increments'].append(int(ang_inc[i]))
# convert stack to spider stack
spiderstack = os.path.join(self.params['rundir'], 'start.spi')
operations.stackToSpiderStack(self.stack['file'],
spiderstack,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
numpart=self.params['numpart'])
# create filtered stack
spiderstackfilt = spiderstack
if (self.params['lowpass'] + self.params['highpass']) > 0:
spiderstackfilt = os.path.join(self.params['rundir'],
'start_filt.spi')
operations.stackToSpiderStack(self.stack['file'],
spiderstackfilt,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
lp=self.params['lowpass'],
hp=self.params['highpass'],
numpart=self.params['numpart'])
# rescale initial model if necessary
outvol = os.path.join(self.params['rundir'], "vol000.spi")
apModel.rescaleModel(self.params['modelid'],
outvol,
self.stack['boxsize'],
self.stack['apix'],
spider=True)
self.params['itervol'] = outvol
for iter in range(1, self.params['numiter'] + 1):
# create projections for projection matching
apDisplay.printMsg("creating reference projections of volume: %s" %
self.params['itervol'])
projs, numprojs, ang, sel = refine.createProjections(
incr=self.params['increments'][iter - 1],
boxsz=self.stack['boxsize'],
symfold=self.params['symfold'],
invol=self.params['itervol'],
rad=self.params['rad'],
)
# run reference-based alignment
apDisplay.printMsg("running reference-based alignment (AP MQ)")
apmqfile = "apmq%03d.spi" % iter
outang = "angular%03d.spi" % iter
shf = "shifts%03d.spi" % iter
shiftedStack = "parts_shifted.spi"
apFile.removeFile(shf)
refine.spiderAPMQ(
projs=projs,
numprojs=numprojs,
tsearch=self.params['xysearch'],
tstep=self.params['xystep'],
firstRing=self.params['firstring'],
lastRing=self.params['lastring'],
stackfile=spiderstackfilt,
nump=self.params['numpart'],
ang=ang,
apmqfile=apmqfile,
outang=outang,
nproc=self.params['proc'],
)
# use cross-correlation to find the sub-pixel alignment
# of the particles,
# results will be saved in "peakfile.spi"
apFile.removeFile(shiftedStack)
# don't use MPI here - for some reason slower?
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
apmqlist = refine.readDocFile(apmqfile)
avgccrot = 0
apDisplay.printMsg("creating shifted stack")
for p in range(0, self.params['numpart']):
ref = int(float(apmqlist[p][2]))
ccrot = float(apmqlist[p][3])
inplane = float(apmqlist[p][4])
avgccrot += ccrot
# invert the sign - ref projs will be rotated
inplane *= -1
# get corresponding projection
if (ref <= 0):
# mirror projection if necessary
ref *= -1
refimg = spyder.fileFilter(projs) + "@" + str(ref)
refine.mirrorImg(refimg, "_3", inMySpi=mySpi)
img = "_3"
else:
img = spyder.fileFilter(projs) + "@" + str(ref)
refine.rotAndShiftImg(img, "_2", inplane, inMySpi=mySpi)
refine.maskImg("_2",
"_3",
self.params['rad'],
"D",
"E",
center=int((self.stack['boxsize'] / 2) + 1),
inMySpi=mySpi)
# pad ref image & stack image to twice the size
refine.padImg("_3",
"_2",
2 * self.stack['boxsize'],
"N",
1,
1,
0,
inMySpi=mySpi)
stackimg = spyder.fileFilter(spiderstack) + "@" + str(p + 1)
refine.padImg(stackimg,
"_1",
2 * self.stack['boxsize'],
"B",
1,
1,
inMySpi=mySpi)
# calculate cross-correlation
refine.getCC("_1", "_2", "_1", inMySpi=mySpi)
# crop the correllation image to allowable shift amount
shift = int(self.params['allowedShift'] *
self.stack['boxsize'])
dim = 2 * shift + 1
topleftx = self.stack['boxsize'] - shift + 1
refine.windowImg("_1",
"_2",
dim,
topleftx,
topleftx,
inMySpi=mySpi)
# find the sub-pixel location of cc peak
mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17", "_2",
"0")
# create new stack of shifted particles
shpos = spyder.fileFilter(shiftedStack) + "@" + str(p + 1)
mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN")
mySpi.toSpiderQuiet("GP x17", "_2",
str(shift + 1) + "," + str(shift + 1))
refine.copyImg(stackimg, shpos, inMySpi=mySpi)
mySpi.toSpiderQuiet("ELSE")
#mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16")
mySpi.toSpiderQuiet("SH F", stackimg, shpos, "-x15,-x16")
mySpi.toSpiderQuiet("ENDIF")
# save shifts to file
mySpi.toSpiderQuiet("SD " + str(p + 1) + ",x15,x16,x17",
spyder.fileFilter(shf))
mySpi.toSpiderQuiet("SD E", spyder.fileFilter(shf))
mySpi.close()
# create class average images
refine.createClassAverages(
shiftedStack,
projs,
apmqfile,
numprojs,
self.stack['boxsize'],
shifted=True,
)
# rename class averages & variacnes for iteration
cmd = "/bin/mv classes.hed classes.%d.hed;" % iter
cmd += "/bin/mv classes.img classes.%d.img;" % iter
cmd += "/bin/mv variances.hed variances.%d.hed;" % iter
cmd += "/bin/mv variances.img variances.%d.img;" % iter
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
# calculate the stddev for the apmq cc's for throwing out particles
avgccrot /= self.params['numpart']
stdccrot = 0
for p in range(0, self.params['numpart']):
stdccrot += abs(float(apmqlist[p][3]) - avgccrot)
stdccrot /= self.params['numpart']
cccutoff = avgccrot + (stdccrot * self.params['keepsig'])
apDisplay.printMsg("average cc: %f" % avgccrot)
apDisplay.printMsg("setting cutoff to: %f" % cccutoff)
# create new selection file that only has particles with good cc's
selectfile = "select%03d.spi" % iter
apFile.removeFile(selectfile)
mySpi = spyder.SpiderSession(nproc=self.params['proc'],
dataext=".spi",
logo=False,
log=False)
i = 1
for p in range(0, self.params['numpart']):
ccrot = float(apmqlist[p][3])
if ccrot >= cccutoff:
mySpi.toSpiderQuiet("x11=%d" % (p + 1))
mySpi.toSpiderQuiet("SD %d,x11" % i,
spyder.fileFilter(selectfile))
i += 1
mySpi.close()
# calculate the new 3d structure using centered projections
# and the corrected angles from the angular doc file
apDisplay.printMsg("creating 3d volume")
out_rawvol = "vol_raw%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
selectfile,
rad=self.params['rad'],
ang=outang,
out=out_rawvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['bpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'],
)
else:
refine.backProjection(shiftedStack,
selectfile,
ang=outang,
out=out_rawvol,
sym=sydoc,
nproc=self.params['proc'])
# create even & odd select files
apDisplay.printMsg("creating even/odd volumes")
oddfile = "selectodd%03d.spi" % iter
evenfile = "selecteven%03d.spi" % iter
refine.docSplit(selectfile, oddfile, evenfile)
# get the even & odd volumesa
oddvol = "vol1%03d.spi" % iter
evenvol = "vol2%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
oddfile,
rad=self.params['rad'],
ang=outang,
out=oddvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'])
refine.iterativeBackProjection(
shiftedStack,
evenfile,
rad=self.params['rad'],
ang=outang,
out=evenvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'])
else:
refine.backProjection(
shiftedStack,
oddfile,
ang=outang,
out=oddvol,
sym=sydoc,
nproc=self.params['proc'],
)
refine.backProjection(
shiftedStack,
evenfile,
ang=outang,
out=evenvol,
sym=sydoc,
nproc=self.params['proc'],
)
# calculate the FSC
apDisplay.printMsg("calculating FSC")
fscfile = "fsc%03d.spi" % iter
emanfsc = "fsc.eotest.%d" % iter
refine.calcFSC(oddvol, evenvol, fscfile)
# convert to eman-style fscfile
refine.spiderFSCtoEMAN(fscfile, emanfsc)
# calculate the resolution at 0.5 FSC & write to file
res = apRecon.calcRes(emanfsc, self.stack['boxsize'],
self.stack['apix'])
restxt = "resolution.txt"
if iter == 1 and os.path.isfile(restxt):
os.remove(restxt)
resfile = open(restxt, "a")
resfile.write("iter %d:\t%.3f\n" % (iter, res))
resfile.close()
# filter & normalize the volume to be used as a reference in the next round
outvol = "vol%03d.spi" % iter
emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % (
out_rawvol, outvol, self.stack['apix'], res,
self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# create mrc files of volumes
emancmd = "proc3d %s %s apix=%.3f mask=%d norm" % (
out_rawvol, "threed.%da.mrc" % iter, self.stack['apix'],
self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % (
out_rawvol, "threed.%da.lp.mrc" % iter, self.stack['apix'],
res, self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.param['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# set this model as start for next iteration, remove previous
os.remove(self.params['itervol'])
os.remove(out_rawvol)
self.params['itervol'] = outvol
# clean up directory
apDisplay.printMsg("cleaning up directory")
if os.path.isfile(oddvol):
os.remove(oddvol)
if os.path.isfile(evenvol):
os.remove(evenvol)
if os.path.isfile(ang):
os.remove(ang)
if os.path.isfile(sel):
os.remove(sel)
if os.path.isfile(projs):
os.remove(projs)
if os.path.isfile(oddfile):
os.remove(oddfile)
if os.path.isfile(evenfile):
os.remove(evenfile)
if os.path.isfile(emanfsc) and os.path.isfile(fscfile):
os.remove(fscfile)
if os.path.isfile(shiftedStack):
os.remove(shiftedStack)
os.remove(self.params['itervol'])
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local file
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
a.setValue('append',False)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert') is True
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
# clip not yet implemented
# self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### stack is needed by uploadMaxlike.py
#apFile.removeStack(self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -nref "+str(self.params['numrefs'])
+" -iter "+str(self.params['maxiter'])
+" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp)
+" -psi_step "+str(self.params['psistep'])
)
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-17 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### use student's T distribution
if self.params['student'] is True:
xmippopts += " -student "
### write cluster job file
if self.params['cluster'] is True:
self.writeClusterJobFile()
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
self.estimateIterTime()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using "+str(nproc)+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun+" -np "+str(nproc)+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe+" "+xmippopts
self.writeXmippLog(xmippcmd)
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.clipsize = int(math.floor(self.stack['boxsize']/self.params['bin']/2.0)*self.params['bin']*2.0)
#self.checkRunNamePath()
### convert stack to spider
spiderstack = self.createSpiderFile()
#create template stack
templatestack = self.createTemplateStack()
#create orientation reference
orientref = self.createOrientationReference()
###################################################################
aligntime = time.time()
### create batch file
batchfilepath = self.setupBatchFile(spiderstack, templatestack, orientref)
### run the spider alignment
apDisplay.printColor("Running iterative ref-classification and free-alignment with spider","cyan")
self.runSpiderBatch( batchfilepath, spiderstack )
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
###################################################################
### remove unaligned spider stack
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=False)
### check to be sure files exist
avgfile = os.path.join(self.params['rundir'], "alignstack.spi") #class averages
if not os.path.isfile(avgfile):
apDisplay.printError("Final stack of aligned particles does not exist.")
### convert stacks to imagic
self.convertStack2Imagic("alignstack.spi")
self.convertStack2Imagic("avg.spi")
### make alignment average in mrc format
emancmd = "proc2d avg.spi average.mrc average"
apEMAN.executeEmanCmd(emancmd)
inserttime = time.time()
if self.params['commit'] is True:
apDisplay.printWarning("committing results to DB")
self.runtime = aligntime
self.insertEdIterRun(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def start(self):
self.addToLog('.... Setting up new ISAC job ....')
self.addToLog('.... Making command for stack pre-processing ....')
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### send file to remotehost
tasks = {}
sfhed = self.stack['file'][:-4] + ".hed"
sfimg = self.stack['file'][:-4] + ".img"
tasks = self.addToTasks(
tasks,
"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s"
% (sfhed, self.params['localhost'], self.params['remoterundir'],
"start1.hed"))
tasks = self.addToTasks(
tasks,
"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s"
% (sfimg, self.params['localhost'], self.params['remoterundir'],
"start1.img"))
### write Sparx jobfile: process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
# apParam.runCmd(proccmd, "EMAN", verbose=True)
self.addSimpleCommand('cd %s' % self.params['rundir'])
self.addSimpleCommand(proccmd)
sparxcmd = "sxcpy.py %s %s_1.hdf" % (self.params['localstack'],
self.params['localstack'][:-4])
# apParam.runCmd(sparxcmd, "SPARX", verbose=True)
self.addSimpleCommand(sparxcmd)
self.addSimpleCommand("")
### write Sparx jobfile: run ISAC
for i in range(self.params['generations']):
sparxopts = " %s_%d.hdf" % (os.path.join(
self.params['localstack'][:-4]), (i + 1))
if self.params['ir'] is not None:
sparxopts += " --ir %d" % int(
float(
apRecon.getComponentFromVector(
self.params['ir'], i, splitter=":")))
if self.params['ou'] is not None:
sparxopts += " --ou %d" % int(
float(
apRecon.getComponentFromVector(
self.params['ou'], i, splitter=":")))
if self.params['rs'] is not None:
sparxopts += " --rs %d" % int(
float(
apRecon.getComponentFromVector(
self.params['rs'], i, splitter=":")))
if self.params['ts'] is not None:
sparxopts += " --ts %.1f" % int(
float(
apRecon.getComponentFromVector(
self.params['ts'], i, splitter=":")))
if self.params['xr'] is not None:
sparxopts += " --xr %.1f" % int(
float(
apRecon.getComponentFromVector(
self.params['xr'], i, splitter=":")))
if self.params['yr'] is not None:
sparxopts += " --yr %.1f" % int(
float(
apRecon.getComponentFromVector(
self.params['yr'], i, splitter=":")))
if self.params['maxit'] is not None:
sparxopts += " --maxit %d" % int(
float(
apRecon.getComponentFromVector(
self.params['maxit'], i, splitter=":")))
if self.params['FL'] is not None:
sparxopts += " --FL %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['FL'], i, splitter=":")))
if self.params['FH'] is not None:
sparxopts += " --FH %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['FH'], i, splitter=":")))
if self.params['FF'] is not None:
sparxopts += " --FF %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['FF'], i, splitter=":")))
if self.params['init_iter'] is not None:
sparxopts += " --init_iter %d" % int(
float(
apRecon.getComponentFromVector(
self.params['init_iter'], i, splitter=":")))
if self.params['main_iter'] is not None:
sparxopts += " --main_iter %d" % int(
float(
apRecon.getComponentFromVector(
self.params['main_iter'], i, splitter=":")))
if self.params['iter_reali'] is not None:
sparxopts += " --iter_reali %d" % int(
float(
apRecon.getComponentFromVector(
self.params['iter_reali'], i, splitter=":")))
if self.params['match_first'] is not None:
sparxopts += " --match_first %d" % int(
float(
apRecon.getComponentFromVector(
self.params['match_first'], i, splitter=":")))
if self.params['max_round'] is not None:
sparxopts += " --max_round %d" % int(
float(
apRecon.getComponentFromVector(
self.params['max_round'], i, splitter=":")))
if self.params['match_second'] is not None:
sparxopts += " --match_second %d" % int(
float(
apRecon.getComponentFromVector(
self.params['match_second'], i, splitter=":")))
if self.params['stab_ali'] is not None:
sparxopts += " --stab_ali %d" % int(
float(
apRecon.getComponentFromVector(
self.params['stab_ali'], i, splitter=":")))
if self.params['thld_err'] is not None:
sparxopts += " --thld_err %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['thld_err'], i, splitter=":")))
if self.params['indep_run'] is not None:
sparxopts += " --indep_run %d" % int(
float(
apRecon.getComponentFromVector(
self.params['indep_run'], i, splitter=":")))
if self.params['thld_grp'] is not None:
sparxopts += " --thld_grp %d" % int(
float(
apRecon.getComponentFromVector(
self.params['thld_grp'], i, splitter=":")))
if self.params['img_per_grp'] is not None:
sparxopts += " --img_per_grp %d" % int(
apRecon.getComponentFromVector(self.params['img_per_grp'],
i))
sparxopts += " --generation %d" % (i + 1)
sparxexe = apParam.getExecPath("sxisac.py", die=True)
mpiruncmd = self.mpirun + " -np " + str(
self.params['nproc']) + " " + sparxexe + " " + sparxopts
bn = os.path.basename(self.params['localstack'])[:-4]
e2cmd = "e2proc2d.py %s_%d.hdf %s_%d.hdf --list=\"generation_%d_unaccounted.txt\"" % \
(bn, i+1, bn, i+2, i+1)
self.addSimpleCommand(mpiruncmd)
self.addSimpleCommand(e2cmd)
# print self.tasks
# print self.command_list
# self.writeCommandListToFile()
apParam.dumpParameters(
self.params, "isac-" + self.params['timestamp'] + "-params.pickle")
def start(self):
### get stack parameteres
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackId'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackId'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackId'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### copy stack into working directory
if os.path.isfile(self.stack['file']):
apDisplay.printColor("copying stack into running directoroy",
"cyan")
if self.stack['file'][-4:] == ".img" or self.stack['file'][
-4:] == ".hed":
strippedfile = self.stack['file'][:-4]
else:
strippedfile = self.stack['file']
while os.path.isfile(
os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(
os.path.join(self.params['rundir'], "start.img"))
emancmd = "proc2d "+strippedfile+".hed "+os.path.join(self.params['rundir'], "start.hed ")+\
"first=0 last="+str(self.params['numpart']-1)
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("stack not found in database")
### get template stack parameters
self.templatestack = {}
self.templatestack[
'data'] = appiondata.ApTemplateStackData.direct_query(
self.params['templateStackId'])
self.templatestack['apix'] = self.templatestack['data']['apix']
self.templatestack['boxsize'] = self.templatestack['data']['boxsize']
self.templatestack['file'] = os.path.join(
self.templatestack['data']['path']['path'],
self.templatestack['data']['templatename'])
self.templatestack['numimages'] = self.templatestack['data'][
'numimages']
### copy templates into working directory
if os.path.isfile(self.templatestack['file']):
apDisplay.printColor("copying templates into running directoroy",
"cyan")
ts = os.path.join(self.params['rundir'], "references.img")
while os.path.isfile(ts):
apFile.removeStack(ts)
if self.templatestack['file'][-4:] == ".img" or self.templatestack[
'file'][-4:] == ".hed":
strippedfile = self.templatestack['file'][:-4]
else:
strippedfile = self.templatestack['file']
emancmd = "proc2d " + strippedfile + ".img " + ts
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("template stack not found in database")
### set new pixelsize
if self.params['bin'] is not None and self.params['bin'] != 0:
self.params['apix'] = float(self.stack['apix']) * int(
self.params['bin'])
else:
self.params['apix'] = self.stack['apix']
### scale, low-pass, and high-pass filter stack ... do this with imagic, because it determines the appropriate boxsizes
scalingbatchfile = self.createImagicBatchFileScaling()
preptime = time.time()
proc = subprocess.Popen("chmod 775 " + str(scalingbatchfile),
shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(scalingbatchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(
os.path.join(self.params['rundir'], "prepareStack.log"))
apDisplay.printColor(
"finished IMAGIC in " +
apDisplay.timeString(time.time() - preptime), "cyan")
### set new boxsize, done only after scaling is complete
if self.params['bin'] is not None:
self.params['boxsize'] = apFile.getBoxSize(
os.path.join(self.params['rundir'], "start.hed"))[0]
else:
self.params['boxsize'] = self.stack['boxsize']
### make sure template stack boxsize matches that of the input stack
if self.params['apix'] != self.templatestack['apix'] or self.params[
'boxsize'] != self.templatestack['boxsize']:
self.scaleTemplates()
starttime = time.time()
print self.params
print "... stack pixel size: " + str(self.params['apix'])
print "... stack box size: " + str(self.params['boxsize'])
apDisplay.printColor(
"Running IMAGIC .batch file: See multiReferenceAlignment.log file for details",
"cyan")
### create IMAGIC batch file
batchfile = self.createImagicBatchFileMRA()
### execute IMAGIC batch file
aligntime0 = time.time()
proc = subprocess.Popen("chmod 775 " + str(batchfile), shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(batchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(
os.path.join(self.params['rundir'], "multiReferenceAlignment.log"))
apDisplay.printColor(
"finished IMAGIC in " +
apDisplay.timeString(time.time() - aligntime0), "cyan")
### get particle parameters (shift, rotate, refnum, mirror, ccc)
partparams = self.getParticleParams()
### average stack
alignstack = os.path.join(self.params['rundir'], "alignstack.hed")
apStack.averageStack(alignstack)
### normalize particles (otherwise found problems in viewing with stackviewer)
emancmd = "proc2d " + alignstack + " " + alignstack + ".norm.hed norm"
while os.path.isfile(alignstack + ".norm.img"):
apFile.removeStack(alignstack + ".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(alignstack + ".norm.hed", alignstack)
os.rename(alignstack + ".norm.img", alignstack[:-4] + ".img")
### normalize references
emancmd = "proc2d " + ts + " " + ts + ".norm.hed norm"
while os.path.isfile(ts + ".norm.img"):
apFile.removeStack(ts + ".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(ts + ".norm.hed", ts)
os.rename(ts + ".norm.img", ts[:-4] + ".img")
### remove copied stack
while os.path.isfile(os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(os.path.join(self.params['rundir'],
"start.img"))
### insert run into database
self.insertAlignmentRun(insert=True)
self.insertParticlesIntoDatabase(partparams, insert=True)
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local temp file
proccmd = "proc2d "+self.stack['file']+" temp.hed apix="+str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize)*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
proccmd += " last="+str(self.params['numpart']-1)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### process stack to final file
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
proccmd = "proc2d temp.hed "+self.params['localstack']+" apix="+str(self.stack['apix']*self.params['bin'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
apFile.removeStack("temp.hed")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### convert stack into single spider files
templateselfile = self.initializeTemplates()
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -iter "+str(self.params['maxiter'])
+" -ref "+templateselfile
+" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp)
+" -psi_step "+str(self.params['psistep'])
)
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-18 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using "+str(nproc-1)+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun+" -np "+str(nproc-1)+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe+" "+xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### setup Xmipp command
aligntime = time.time()
xmippopts = (" -i "+os.path.join(self.params['rundir'], self.params['localstack'])
+" --nref "+str(self.params['numrefs'])
+" --iter "+str(self.params['maxiter'])
+" --odir "+str(self.params['rundir'])
+" --oroot "+ "part"+str(self.params['timestamp'])
+" --classifyAllImages"
)
if self.params['correlation']:
xmippopts += " --distance correlation"
if self.params['classical']:
xmippopts += " --classicalMultiref"
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath(self.execFile, die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp 3", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### post-processing
# Create a stack for the class averages at each level
Nlevels=glob.glob("level_*")
for level in Nlevels:
digits = level.split("_")[1]
apParam.runCmd("xmipp_image_convert -i "+level+"/part"+self.params['timestamp']+"*xmd -o part"
+self.params['timestamp']+"_level_"+digits+"_.hed", package="Xmipp 3", verbose=True)
if self.params['align']:
apParam.runCmd("xmipp_transform_geometry -i images.xmd -o %s_aligned.stk --apply_transform" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apParam.runCmd("xmipp_image_convert -i %s_aligned.xmd -o alignedStack.hed" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apFile.removeFile("%s_aligned.xmd" % self.params['timestamp'])
apFile.removeFile("%s_aligned.stk" % self.params['timestamp'])
self.parseOutput()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
self.clearIntermediateFiles()
def start(self):
if self.params['rundir'] is None or not os.path.isdir(
self.params['rundir']):
apDisplay.printError("upload directory does not exist: " +
str(self.params['rundir']))
### create temp directory for extracting data
self.params['tmpdir'] = os.path.join(self.params['rundir'], "temp")
apParam.createDirectory(self.params['tmpdir'], warning=True)
### make sure that the stack & model IDs exist in database
emanJobFile = self.findEmanJobFile()
self.params['stack'] = apStack.getOnlyStackData(self.params['stackid'])
self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
self.params['model'] = appiondata.ApInitialModelData.direct_query(
self.params['modelid'])
self.params['boxsize'] = apStack.getStackBoxsize(
self.params['stackid'])
### parse out the refinement parameters from the log file
self.parseLogFile()
### parse out the message passing subclassification parameters from the job/log file
if self.params['package'] == 'EMAN/MsgP':
self.parseMsgPassingParams()
### convert class average files from old to new format
self.convertClassAvgFiles()
### get a list of the files in the directory
self.listFiles()
### create a refinementRun entry in the database
self.insertRefinementRun()
if self.params['euleronly'] is False:
### insert the Iteration info
for iteration in self.iterationdatas:
### if only uploading one iteration, skip to that one
if self.params['oneiter'] and int(
iteration['num']) != self.params['oneiter']:
continue
### if beginning at later iteration, skip to that one
if self.params['startiter'] and int(
iteration['num']) < self.params['startiter']:
continue
### if beginning at later iteration, skip to that one
if self.params['enditer'] and int(
iteration['num']) > self.params['enditer']:
continue
apDisplay.printColor(
"\nUploading iteration " + str(iteration['num']) + " of " +
str(len(self.iterationdatas)) + "\n", "green")
for i in range(75):
sys.stderr.write("#")
sys.stderr.write("\n")
self.insertIteration(iteration)
### calculate euler jumps
if self.params['commit'] is True:
reconrunid = self.params['refineRun'].dbid
stackid = self.params['stack'].dbid
if self.params['oneiter'] is None and len(self.iterationdatas) > 1:
apDisplay.printMsg("calculating euler jumpers for recon=" +
str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(
reconrunid, stackid)
### coran keep plot
if self.params['package'] == 'EMAN/SpiCoran':
apCoranPlot.makeCoranKeepPlot(reconrunid)
apRecon.setGoodBadParticlesFromReconId(reconrunid)
def makeStackMeanPlot(stackid, gridpoints=16):
if gridpoints > 30:
apDisplay.printError("Too large of a grid")
apDisplay.printMsg("creating Stack Mean Plot montage for stackid: "+str(stackid))
t0 = time.time()
### big stacks are too slow
boxsize = apStack.getStackBoxsize(stackid)
bin = 1
if boxsize is not None:
while boxsize/bin > 128:
bin+=1
apDisplay.printMsg("binning stack by "+str(bin))
stackdata = apStack.getOnlyStackData(stackid, msg=False)
stackpath = stackdata['path']['path']
stackfile = os.path.join(stackpath, stackdata['name'])
# if no stackfile, likely virtual stack
if not os.path.isfile(stackfile):
apDisplay.printMsg("possible virtual stack, searching for original stack")
vstackdata = apStack.getVirtualStackParticlesFromId(stackid)
partdatas = vstackdata['particles']
stackfile = vstackdata['filename']
stackdata = apStack.getOnlyStackData(vstackdata['stackid'], msg=False)
# otherwise get stack info
else:
# get stats from stack:
sqlcmd = "SELECT " + \
"particleNumber, mean, stdev " + \
"FROM ApStackParticleData " + \
"WHERE `REF|ApStackData|stack` = %i"%(stackid)
partdatas = sinedon.directq.complexMysqlQuery('appiondata',sqlcmd)
#check only first 100 particles for now
#partdatas = partdatas[:500]
apFile.removeFile("montage"+str(stackid)+".png")
### find limits
limits = {'minmean': 1e12, 'maxmean': -1e12, 'minstdev': 1e12, 'maxstdev': -1e12,}
for partdata in partdatas:
if partdata['mean'] is None:
continue
mean = partdata['mean']
stdev = partdata['stdev']
if mean < limits['minmean']:
limits['minmean'] = mean
if mean > limits['maxmean']:
limits['maxmean'] = mean
if stdev < limits['minstdev']:
limits['minstdev'] = stdev
if stdev > limits['maxstdev']:
limits['maxstdev'] = stdev
if limits['minmean'] > 1e11:
apDisplay.printWarning("particles have no mean values in database")
return
apDisplay.printMsg(str(limits))
### create particle bins
partlists = {}
for i in range(gridpoints):
for j in range(gridpoints):
key = ("%02dx%02d"%(i,j))
partlists[key] = []
### sort particles into bins
for partdata in partdatas:
key = meanStdevToKey(partdata['mean'], partdata['stdev'], limits, gridpoints)
partnum = int(partdata['particleNumber'])
partlists[key].append(partnum)
printPlot(partlists, gridpoints)
### createStackAverages
keys = partlists.keys()
keys.sort()
count = 0
backs = "\b\b\b\b\b\b\b\b\b\b\b"
montagestack = os.path.join(stackpath,"montage"+str(stackid)+".hed")
apFile.removeStack(montagestack)
mystack = []
for key in keys:
count += 1
sys.stderr.write(backs+backs+backs+backs)
sys.stderr.write("% 3d of % 3d, %s: % 6d"%(count, len(keys), key, len(partlists[key])))
avgimg = averageSubStack(partlists[key], stackfile, bin)
if avgimg is not False:
avgimg = numpy.fliplr(avgimg)
mystack.append(avgimg)
apImagicFile.writeImagic(mystack, montagestack)
sys.stderr.write("\n")
assemblePngs(keys, str(stackid), montagestack)
apDisplay.printMsg("/bin/mv -v montage"+str(stackid)+".??? "+stackpath)
apDisplay.printMsg("finished in "+apDisplay.timeString(time.time()-t0))
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### setup Xmipp command
aligntime = time.time()
xmippopts = (" -i "+os.path.join(self.params['rundir'], self.params['localstack'])
+" --nref "+str(self.params['numrefs'])
+" --iter "+str(self.params['maxiter'])
+" --odir "+str(self.params['rundir'])
+" --oroot "+ "part"+str(self.params['timestamp'])
+" --classifyAllImages"
)
if self.params['correlation']:
xmippopts += " --distance correlation"
if self.params['classical']:
xmippopts += " --classicalMultiref"
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath(self.execFile, die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp 3", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### post-processing
# Create a stack for the class averages at each level
Nlevels=glob.glob("level_*")
for level in Nlevels:
digits = level.split("_")[1]
apParam.runCmd("xmipp_image_convert -i "+level+"/part"+self.params['timestamp']+"*xmd -o part"
+self.params['timestamp']+"_level_"+digits+"_.hed", package="Xmipp 3", verbose=True)
if self.params['align']:
apParam.runCmd("xmipp_transform_geometry -i images.xmd -o %s_aligned.stk --apply_transform" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apParam.runCmd("xmipp_image_convert -i %s_aligned.xmd -o alignedStack.hed" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apFile.removeFile("%s_aligned.xmd" % self.params['timestamp'])
apFile.removeFile("%s_aligned.stk" % self.params['timestamp'])
self.parseOutput()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
self.clearIntermediateFiles()
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local temp file
proccmd = "proc2d " + self.stack['file'] + " temp.hed apix=" + str(
self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### process stack to final file
self.params['localstack'] = os.path.join(self.params['rundir'],
self.timestamp + ".hed")
proccmd = "proc2d temp.hed " + self.params[
'localstack'] + " apix=" + str(
self.stack['apix'] * self.params['bin'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
apFile.removeStack("temp.hed")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### convert stack into single spider files
templateselfile = self.initializeTemplates()
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -iter " + str(self.params['maxiter']) + " -ref " +
templateselfile + " -o " +
os.path.join(self.params['rundir'], "part" + self.timestamp) +
" -psi_step " + str(self.params['psistep']))
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-18 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using " + str(nproc - 1) + " processors!",
"green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun + " -np " + str(
nproc - 1) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.clipsize = int(
math.floor(self.stack['boxsize'] / self.params['bin'] / 2.0) *
self.params['bin'] * 2.0)
#self.checkRunNamePath()
### convert stack to spider
spiderstack = self.createSpiderFile()
#create template stack
templatestack = self.createTemplateStack()
#create orientation reference
orientref = self.createOrientationReference()
###################################################################
aligntime = time.time()
### create batch file
batchfilepath = self.setupBatchFile(spiderstack, templatestack,
orientref)
### run the spider alignment
apDisplay.printColor(
"Running iterative ref-classification and free-alignment with spider",
"cyan")
self.runSpiderBatch(batchfilepath, spiderstack)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
###################################################################
### remove unaligned spider stack
apDisplay.printMsg("Removing un-aligned stack: " + spiderstack)
apFile.removeFile(spiderstack, warn=False)
### check to be sure files exist
avgfile = os.path.join(self.params['rundir'],
"alignstack.spi") #class averages
if not os.path.isfile(avgfile):
apDisplay.printError(
"Final stack of aligned particles does not exist.")
### convert stacks to imagic
self.convertStack2Imagic("alignstack.spi")
self.convertStack2Imagic("avg.spi")
### make alignment average in mrc format
emancmd = "proc2d avg.spi average.mrc average"
apEMAN.executeEmanCmd(emancmd)
inserttime = time.time()
if self.params['commit'] is True:
apDisplay.printWarning("committing results to DB")
self.runtime = aligntime
self.insertEdIterRun(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: " +
apDisplay.timeString(inserttime))
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
# symmetry info
if self.params['sym']=='Icos':
self.params['symtype']='I'
self.params['symfold']=None
else:
self.params['symtype']=self.params['sym'][0]
self.params['symfold']=int(self.params['sym'][1:])
# eman "d" symmetry is spider "ci"
if self.params['symtype'].upper()=="D":
self.params['symtype'] = "CI"
# create symmetry doc file
sydoc="sym.spi"
refine.symmetryDoc(self.params['symtype'],self.params['symfold'],sydoc)
# convert incr to array of increments
ang_inc=self.params['incr'].split(',')
self.params['numiter'] = len(ang_inc)
self.params['increments']=[]
for i in range(0,self.params['numiter']):
self.params['increments'].append(int(ang_inc[i]))
# convert stack to spider stack
spiderstack=os.path.join(self.params['rundir'],'start.spi')
operations.stackToSpiderStack(
self.stack['file'],
spiderstack,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
numpart=self.params['numpart']
)
# create filtered stack
spiderstackfilt=spiderstack
if (self.params['lowpass']+self.params['highpass']) > 0:
spiderstackfilt=os.path.join(self.params['rundir'],'start_filt.spi')
operations.stackToSpiderStack(
self.stack['file'],
spiderstackfilt,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
lp=self.params['lowpass'],
hp=self.params['highpass'],
numpart=self.params['numpart']
)
# rescale initial model if necessary
outvol = os.path.join(self.params['rundir'],"vol000.spi")
apModel.rescaleModel(self.params['modelid'], outvol, self.stack['boxsize'], self.stack['apix'], spider=True)
self.params['itervol']=outvol
for iter in range(1,self.params['numiter']+1):
# create projections for projection matching
apDisplay.printMsg("creating reference projections of volume: %s" % self.params['itervol'])
projs,numprojs,ang,sel = refine.createProjections(
incr=self.params['increments'][iter-1],
boxsz=self.stack['boxsize'],
symfold=self.params['symfold'],
invol=self.params['itervol'],
rad=self.params['rad'],
)
# run reference-based alignment
apDisplay.printMsg("running reference-based alignment (AP MQ)")
apmqfile = "apmq%03d.spi" % iter
outang = "angular%03d.spi" % iter
shf = "shifts%03d.spi" % iter
shiftedStack="parts_shifted.spi"
apFile.removeFile(shf)
refine.spiderAPMQ(
projs=projs,
numprojs=numprojs,
tsearch=self.params['xysearch'],
tstep=self.params['xystep'],
firstRing=self.params['firstring'],
lastRing=self.params['lastring'],
stackfile=spiderstackfilt,
nump=self.params['numpart'],
ang=ang,
apmqfile=apmqfile,
outang=outang,
nproc=self.params['proc'],
)
# use cross-correlation to find the sub-pixel alignment
# of the particles,
# results will be saved in "peakfile.spi"
apFile.removeFile(shiftedStack)
# don't use MPI here - for some reason slower?
mySpi=spyder.SpiderSession(dataext=".spi", logo=False, log=False)
apmqlist = refine.readDocFile(apmqfile)
avgccrot = 0
apDisplay.printMsg("creating shifted stack")
for p in range(0,self.params['numpart']):
ref=int(float(apmqlist[p][2]))
ccrot=float(apmqlist[p][3])
inplane=float(apmqlist[p][4])
avgccrot+=ccrot
# invert the sign - ref projs will be rotated
inplane*=-1
# get corresponding projection
if (ref <= 0):
# mirror projection if necessary
ref*=-1
refimg=spyder.fileFilter(projs)+"@"+str(ref)
refine.mirrorImg(refimg,"_3",inMySpi=mySpi)
img="_3"
else:
img=spyder.fileFilter(projs)+"@"+str(ref)
refine.rotAndShiftImg(img,"_2",inplane,inMySpi=mySpi)
refine.maskImg("_2","_3",self.params['rad'],"D","E",
center=int((self.stack['boxsize']/2)+1),
inMySpi=mySpi)
# pad ref image & stack image to twice the size
refine.padImg("_3","_2",2*self.stack['boxsize'],"N",1,1,0,inMySpi=mySpi)
stackimg=spyder.fileFilter(spiderstack)+"@"+str(p+1)
refine.padImg(stackimg,"_1",2*self.stack['boxsize'],"B",1,1,inMySpi=mySpi)
# calculate cross-correlation
refine.getCC("_1","_2","_1",inMySpi=mySpi)
# crop the correllation image to allowable shift amount
shift=int(self.params['allowedShift']*self.stack['boxsize'])
dim=2*shift+1
topleftx=self.stack['boxsize']-shift+1
refine.windowImg("_1","_2",dim,topleftx,topleftx,inMySpi=mySpi)
# find the sub-pixel location of cc peak
mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17","_2","0")
# create new stack of shifted particles
shpos=spyder.fileFilter(shiftedStack)+"@"+str(p+1)
mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN")
mySpi.toSpiderQuiet("GP x17","_2",str(shift+1)+","+str(shift+1))
refine.copyImg(stackimg,shpos,inMySpi=mySpi)
mySpi.toSpiderQuiet("ELSE")
#mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16")
mySpi.toSpiderQuiet("SH F",stackimg,shpos,"-x15,-x16")
mySpi.toSpiderQuiet("ENDIF")
# save shifts to file
mySpi.toSpiderQuiet("SD "+str(p+1)+",x15,x16,x17",spyder.fileFilter(shf))
mySpi.toSpiderQuiet("SD E",spyder.fileFilter(shf))
mySpi.close()
# create class average images
refine.createClassAverages(
shiftedStack,
projs,
apmqfile,
numprojs,
self.stack['boxsize'],
shifted=True,
)
# rename class averages & variacnes for iteration
cmd="/bin/mv classes.hed classes.%d.hed;" % iter
cmd+="/bin/mv classes.img classes.%d.img;" % iter
cmd+="/bin/mv variances.hed variances.%d.hed;" % iter
cmd+="/bin/mv variances.img variances.%d.img;" % iter
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
# calculate the stddev for the apmq cc's for throwing out particles
avgccrot/=self.params['numpart']
stdccrot = 0
for p in range(0,self.params['numpart']):
stdccrot+=abs(float(apmqlist[p][3])-avgccrot)
stdccrot/=self.params['numpart']
cccutoff=avgccrot+(stdccrot*self.params['keepsig'])
apDisplay.printMsg("average cc: %f" %avgccrot)
apDisplay.printMsg("setting cutoff to: %f" %cccutoff)
# create new selection file that only has particles with good cc's
selectfile="select%03d.spi" % iter
apFile.removeFile(selectfile)
mySpi = spyder.SpiderSession(nproc=self.params['proc'],dataext=".spi", logo=False, log=False)
i=1
for p in range(0,self.params['numpart']):
ccrot=float(apmqlist[p][3])
if ccrot>=cccutoff:
mySpi.toSpiderQuiet("x11=%d" % (p+1))
mySpi.toSpiderQuiet("SD %d,x11" % i,spyder.fileFilter(selectfile))
i+=1
mySpi.close()
# calculate the new 3d structure using centered projections
# and the corrected angles from the angular doc file
apDisplay.printMsg("creating 3d volume")
out_rawvol="vol_raw%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
selectfile,
rad=self.params['rad'],
ang=outang,
out=out_rawvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['bpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'],
)
else:
refine.backProjection(
shiftedStack,
selectfile,
ang=outang,
out=out_rawvol,
sym=sydoc,
nproc=self.params['proc']
)
# create even & odd select files
apDisplay.printMsg("creating even/odd volumes")
oddfile="selectodd%03d.spi" % iter
evenfile="selecteven%03d.spi" % iter
refine.docSplit(selectfile,oddfile,evenfile)
# get the even & odd volumesa
oddvol="vol1%03d.spi" % iter
evenvol="vol2%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
oddfile,
rad=self.params['rad'],
ang=outang,
out=oddvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc']
)
refine.iterativeBackProjection(
shiftedStack,
evenfile,
rad=self.params['rad'],
ang=outang,
out=evenvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc']
)
else:
refine.backProjection(
shiftedStack,
oddfile,
ang=outang,
out=oddvol,
sym=sydoc,
nproc=self.params['proc'],
)
refine.backProjection(
shiftedStack,
evenfile,
ang=outang,
out=evenvol,
sym=sydoc,
nproc=self.params['proc'],
)
# calculate the FSC
apDisplay.printMsg("calculating FSC")
fscfile="fsc%03d.spi" % iter
emanfsc="fsc.eotest.%d" % iter
refine.calcFSC(oddvol,evenvol,fscfile)
# convert to eman-style fscfile
refine.spiderFSCtoEMAN(fscfile,emanfsc)
# calculate the resolution at 0.5 FSC & write to file
res=apRecon.calcRes(emanfsc,self.stack['boxsize'],self.stack['apix'])
restxt="resolution.txt"
if iter==1 and os.path.isfile(restxt):
os.remove(restxt)
resfile=open(restxt,"a")
resfile.write("iter %d:\t%.3f\n" % (iter,res))
resfile.close()
# filter & normalize the volume to be used as a reference in the next round
outvol="vol%03d.spi" % iter
emancmd="proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % (out_rawvol,outvol,self.stack['apix'],res,self.params['rad'])
if self.params['imask'] is not None:
emancmd+=" imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# create mrc files of volumes
emancmd="proc3d %s %s apix=%.3f mask=%d norm" % (out_rawvol,"threed.%da.mrc" % iter, self.stack['apix'],self.params['rad'])
if self.params['imask'] is not None:
emancmd+=" imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
emancmd="proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % (out_rawvol,"threed.%da.lp.mrc" % iter, self.stack['apix'],res, self.params['rad'])
if self.params['imask'] is not None:
emancmd+=" imask=%d" % self.param['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# set this model as start for next iteration, remove previous
os.remove(self.params['itervol'])
os.remove(out_rawvol)
self.params['itervol']=outvol
# clean up directory
apDisplay.printMsg("cleaning up directory")
if os.path.isfile(oddvol):
os.remove(oddvol)
if os.path.isfile(evenvol):
os.remove(evenvol)
if os.path.isfile(ang):
os.remove(ang)
if os.path.isfile(sel):
os.remove(sel)
if os.path.isfile(projs):
os.remove(projs)
if os.path.isfile(oddfile):
os.remove(oddfile)
if os.path.isfile(evenfile):
os.remove(evenfile)
if os.path.isfile(emanfsc) and os.path.isfile(fscfile):
os.remove(fscfile)
if os.path.isfile(shiftedStack):
os.remove(shiftedStack)
os.remove(self.params['itervol'])
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
### test insert to make sure data is not overwritten
self.params['runtime'] = 0
#self.checkDuplicateRefBasedRun()
### set box size
self.boxsize = int(math.floor(self.stack['boxsize']/self.params['bin']/2.0))*2
### convert stack to spider
spiderstack = self.createSpiderFile()
### create template stack
templatestack = self.createTemplateStack()
### run the alignment
aligntime = time.time()
usestack = spiderstack
oldpartlist = None
for i in range(self.params['numiter']):
iternum = i+1
apDisplay.printColor("\n\nITERATION "+str(iternum), "green")
alignedstack, partlist = self.runAlignmentGPU(
usestack, templatestack, spiderstack,
self.params['xysearch'], self.params['xystep'],
self.params['firstring'], self.params['lastring'],
iternum=iternum, oldpartlist=oldpartlist)
oldpartlist = partlist
usestack = alignedstack
templatestack = self.updateTemplateStack(alignedstack, partlist, iternum)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=True)
### convert aligned stack to imagic
finalspistack = "aligned.spi"
shutil.move(alignedstack, finalspistack)
imagicstack = "aligned.hed"
apFile.removeStack(imagicstack)
emancmd = "proc2d "+finalspistack+" "+imagicstack
apEMAN.executeEmanCmd(emancmd, verbose=True)
### average stack
apStack.averageStack(imagicstack)
### calculate resolution for each reference
apix = self.stack['apix']*self.params['bin']
self.calcResolution(partlist, imagicstack, apix)
if self.params['commit'] is True:
apDisplay.printMsg("committing results to DB")
self.params['runtime'] = aligntime
self.insertRefBasedRun(partlist, imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
### remove temporary files
apFile.removeFilePattern("alignments/alignedstack*.spi")
apFile.removeFile(finalspistack)
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
### test insert to make sure data is not overwritten
self.params['runtime'] = 0
#self.checkDuplicateRefBasedRun()
### set box size
self.boxsize = int(math.floor(self.stack['boxsize']/self.params['bin']/2.0))*2
### convert stack to spider
spiderstack = self.createSpiderFile()
### create template stack
templatestack = self.createTemplateStack()
### run the alignment
aligntime = time.time()
usestack = spiderstack
oldpartlist = None
for i in range(self.params['numiter']):
iternum = i+1
apDisplay.printColor("\n\nITERATION "+str(iternum), "green")
alignedstack, partlist = self.runAlignmentGPU(
usestack, templatestack, spiderstack,
self.params['xysearch'], self.params['xystep'],
self.params['firstring'], self.params['lastring'],
iternum=iternum, oldpartlist=oldpartlist)
oldpartlist = partlist
usestack = alignedstack
templatestack = self.updateTemplateStack(alignedstack, partlist, iternum)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=True)
### convert aligned stack to imagic
finalspistack = "aligned.spi"
shutil.move(alignedstack, finalspistack)
imagicstack = "aligned.hed"
apFile.removeStack(imagicstack)
emancmd = "proc2d "+finalspistack+" "+imagicstack
apEMAN.executeEmanCmd(emancmd, verbose=True)
### average stack
apStack.averageStack(imagicstack)
### calculate resolution for each reference
apix = self.stack['apix']*self.params['bin']
self.calcResolution(partlist, imagicstack, apix)
if self.params['commit'] is True:
apDisplay.printMsg("committing results to DB")
self.params['runtime'] = aligntime
self.insertRefBasedRun(partlist, imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
### remove temporary files
apFile.removeFilePattern("alignments/alignedstack*.spi")
apFile.removeFile(finalspistack)
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(
self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '" + newdbname + "'",
"green")
### get stack data
self.stack = {}
if self.params['stackid'] is not None:
self.stack['data'] = apStack.getOnlyStackData(
self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(
self.params['stackid'])
self.stack['file'] = os.path.join(
self.stack['data']['path']['path'], self.stack['data']['name'])
else:
self.stack['data'] = appiondata.ApAlignStackData.direct_query(
self.params['alignstackid'])
self.stack['apix'] = self.stack['pixelsize']
self.stack['boxsize'] = self.stack['boxsize']
self.stack['file'] = self.stack['imagicfile']
### check conflicts
if self.params['stackid'] is None and self.params[
'alignstackid'] is None:
apDisplay.printError("stack id OR alignstack id was not defined")
if self.params['stackid'] is not None and self.params[
'alignstackid'] is not None:
apDisplay.printError(
"either specify stack id OR alignstack id, not both")
if self.params['generations'] is None:
apDisplay.printError("number of generations was not provided")
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
if self.params['numpart'] > apFile.numImagesInStack(
self.stack['file']):
apDisplay.printError(
"trying to use more particles " + str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(self.stack['file'])))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(
self.stack['file'])
boxsize = apStack.getStackBoxsize(self.params['stackid'])
if self.params['ou'] is None:
self.params['ou'] = (boxsize / 2.0) - 2
self.clipsize = int(math.floor(
boxsize / float(self.params['bin'] * 2))) * 2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d" %
(self.params['clipsize'], self.clipsize))
self.clipsize = self.params['clipsize']
self.mpirun = self.checkMPI()
if self.mpirun is None:
apDisplay.printError("There is no MPI installed")
def checkConflicts(self):
### first get all stack data
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### modify boxsize and pixelsize according to binning
self.boxsize = self.stack['boxsize']
self.clipsize = int(
math.floor(self.boxsize / float(self.params['bin'] * 2))) * 2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d" %
(self.params['clipsize'], self.clipsize))
self.clipsize = self.params['clipsize']
self.apix = self.stack['apix']
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
self.boxsize = clipsize
self.apix = self.apix * self.params['bin']
### basic error checking
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['ncls'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 1000000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles " +
str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(stackfile)))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > 5000:
apDisplay.printWarning(
"initial model calculation may not work with less than 5000 particles"
)
self.mpirun = self.checkMPI()
# if self.mpirun is None:
# apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
### SIMPLE defaults and error checking
if self.params['ring2'] is None:
self.params['ring2'] = (self.boxsize / 2) - 2
if self.params['ncls'] > 2000:
apDisplay.printError(
"number of classes should be less than 2000 for subsequent ORIGAMI run to work"
)
if self.params['ncls'] > self.params['numpart']:
self.params['ncls'] = self.params['numpart'] / self.params['minp']
if self.params['mask'] is None:
self.params['mask'] = (self.boxsize / 2) - 2
if self.params['mw'] is None:
apDisplay.printError(
"please specify the molecular weight (in kDa)")
def checkConflicts(self):
### first get all stack data
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
### modify boxsize and pixelsize according to binning
self.boxsize = self.stack['boxsize']
self.clipsize = int(math.floor(self.boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
self.apix = self.stack['apix']
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize)*self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
self.boxsize = clipsize
self.apix = self.apix * self.params['bin']
### basic error checking
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['ncls'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 1000000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > 5000:
apDisplay.printWarning("initial model calculation may not work with less than 5000 particles")
self.mpirun = self.checkMPI()
# if self.mpirun is None:
# apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
### SIMPLE defaults and error checking
if self.params['ring2'] is None:
self.params['ring2'] = (self.boxsize/2) - 2
if self.params['ncls'] > 2000:
apDisplay.printError("number of classes should be less than 2000 for subsequent ORIGAMI run to work")
if self.params['ncls'] > self.params['numpart']:
self.params['ncls'] = self.params['numpart'] / self.params['minp']
if self.params['mask'] is None:
self.params['mask'] = (self.boxsize/2) - 2
if self.params['mw'] is None:
apDisplay.printError("please specify the molecular weight (in kDa)")
