def checkConflicts(self):
if self.params["stackid"] is None:
apDisplay.printError("stack id was not defined")
if self.params["modelid"] is None:
apDisplay.printError("model id was not defined")
if self.params["runname"] is None:
apDisplay.printError("missing a reconstruction name")
if self.params["last"] is None:
self.params["last"] = apStack.getNumberStackParticlesFromId(self.params["stackid"])
self.boxsize = apStack.getStackBoxsize(self.params["stackid"], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
if self.params["reconstackid"] is not None:
reconboxsize = apStack.getStackBoxsize(self.params["reconstackid"], msg=False)
reconapix = apStack.getStackPixelSizeFromStackId(self.params["reconstackid"])
refinenumpart = apStack.getNumberStackParticlesFromId(self.params["stackid"])
reconnumpart = apStack.getNumberStackParticlesFromId(self.params["reconstackid"])
if reconboxsize != self.boxsize:
apDisplay.printError("Boxsize do not match for stacks")
if reconapix != self.apix:
apDisplay.printError("Pixelsize do not match for stacks")
if refinenumpart != reconnumpart:
apDisplay.printError("Number of particles do not match for stacks")
maxmask = math.floor(self.apix * (self.boxsize - 10) / 2.0)
if self.params["mask"] is None:
apDisplay.printWarning("mask was not defined, setting to boxsize: %d" % (maxmask))
self.params["mask"] = maxmask
if self.params["mask"] > maxmask:
apDisplay.printWarning("mask was too big, setting to boxsize: %d" % (maxmask))
self.params["mask"] = maxmask
if self.params["noctf"] is True:
apDisplay.printWarning("Using no CTF method")
self.params["wgh"] = -1.0
if self.params["nodes"] > 1 and self.params["cluster"] is False:
apDisplay.printError("cluster mode must be enabled to run on more than 1 node")
if self.params["ppn"] is None:
if self.params["cluster"] is True:
apDisplay.printError("you must define ppn for cluster mode")
self.params["ppn"] = apParam.getNumProcessors()
apDisplay.printMsg("Setting number of processors to %d" % (self.params["ppn"]))
if self.params["rpn"] is None:
self.params["rpn"] = self.params["ppn"]
self.params["nproc"] = self.params["nodes"] * self.params["rpn"]
### get the symmetry data
if self.params["sym"] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params["sym"])
self.params["symm_id"] = self.symmdata.dbid
self.params["symm_name"] = self.symmdata["eman_name"]
apDisplay.printMsg("Selected symmetry %s with id %s" % (self.symmdata["eman_name"], self.symmdata.dbid))
### set cs value
self.params["cs"] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['runname'] is None:
apDisplay.printError("missing a reconstruction name")
if self.params['last'] is None:
self.params['last'] = apStack.getNumberStackParticlesFromId(self.params['stackid'])
self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
if self.params['reconstackid'] is not None:
reconboxsize = apStack.getStackBoxsize(self.params['reconstackid'], msg=False)
reconapix = apStack.getStackPixelSizeFromStackId(self.params['reconstackid'])
refinenumpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
reconnumpart = apStack.getNumberStackParticlesFromId(self.params['reconstackid'])
if reconboxsize != self.boxsize:
apDisplay.printError("Boxsize do not match for stacks")
if reconapix != self.apix:
apDisplay.printError("Pixelsize do not match for stacks")
if refinenumpart != reconnumpart:
apDisplay.printError("Number of particles do not match for stacks")
maxmask = math.floor(self.apix*(self.boxsize-10)/2.0)
if self.params['mask'] is None:
apDisplay.printWarning("mask was not defined, setting to boxsize: %d"%(maxmask))
self.params['mask'] = maxmask
if self.params['mask'] > maxmask:
apDisplay.printWarning("mask was too big, setting to boxsize: %d"%(maxmask))
self.params['mask'] = maxmask
if self.params['noctf'] is True:
apDisplay.printWarning("Using no CTF method")
self.params['wgh'] = -1.0
if self.params['nodes'] > 1 and self.params['cluster'] is False:
apDisplay.printError("cluster mode must be enabled to run on more than 1 node")
if self.params['ppn'] is None:
if self.params['cluster'] is True:
apDisplay.printError("you must define ppn for cluster mode")
self.params['ppn'] = apParam.getNumProcessors()
apDisplay.printMsg("Setting number of processors to %d"%(self.params['ppn']))
if self.params['rpn'] is None:
self.params['rpn'] = self.params['ppn']
self.params['nproc'] = self.params['nodes']*self.params['rpn']
### get the symmetry data
if self.params['sym'] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params['sym'])
self.params['symm_id'] = self.symmdata.dbid
self.params['symm_name'] = self.symmdata['eman_name']
apDisplay.printMsg("Selected symmetry %s with id %s"%(self.symmdata['eman_name'], self.symmdata.dbid))
### set cs value
self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
def medianVolume(self):
volpath = os.path.join(self.params['rundir'], "volumes/*a.mrc")
mrcfiles = glob.glob(volpath)
volumes = []
for filename in mrcfiles:
if os.path.isfile(filename):
vol = mrc.read(filename)
print filename, vol.shape
volumes.append(vol)
volarray = numpy.asarray(volumes, dtype=numpy.float32)
try:
medarray = numpy.median(volarray, axis=0)
except:
medarray = numpy.median(volarray)
medfile = os.path.join(self.params['rundir'], "volumes/medianVolume.mrc")
print medfile, medarray.shape
mrc.write(medarray, medfile)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
uploadcmd = ( ("uploadModel.py --projectid=%d --session=%s --file=%s "
+"--apix=%.3f --sym=%s --name=satmedian-recon%d.mrc --res=30 --description='%s %d'")
%(self.params['projectid'], sessiondata['name'], medfile,
apix, self.params['symmname'], self.params['reconid'],
"SAT selected median volume for recon", self.params['reconid'], ) )
apDisplay.printColor(uploadcmd, "purple")
f = open("upload.sh", "w")
f.write(uploadcmd+"\n")
f.close()
def setupParticles(self):
self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed")
self.stack = {}
self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"])
self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"])
self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"])
self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"])
boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2
apix = self.stack["apix"] * self.params["bin"]
proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"])
if self.params["bin"] > 1:
clipsize = boxsize * self.params["bin"]
proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize)
if self.params["highpass"] > 1:
proccmd += " hp=" + str(self.params["highpass"])
if self.params["lowpass"] > 1:
proccmd += " lp=" + str(self.params["lowpass"])
proccmd += " last=" + str(self.params["numpart"] - 1)
apFile.removeStack(self.params["localstack"], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"])
return (boxsize, apix)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("Please provide a user id, e.g. --stackid=15")
if not 'radonshift' in dir(radermacher):
apDisplay.printError("Radermacher python module is too old, please update")
boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
pixelsize = apStack.getStackPixelSizeFromStackId(self.params['stackid'], msg=False)
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'], msg=False)
apDisplay.printMsg("\n\n")
### get info about the stack
apDisplay.printMsg("Information about stack id %d"%(self.params['stackid']))
apDisplay.printMsg("\tboxsize %d pixels"%(boxsize))
apDisplay.printMsg("\tpixelsize %.3f Angstroms"%(pixelsize))
apDisplay.printMsg("\tsize %d particles"%(numpart))
if self.params['maskrad'] is None:
self.params['pixelmaskrad'] = boxsize/2
self.params['maskrad'] = self.params['pixelmaskrad']*pixelsize
else:
self.params['pixelmaskrad'] = self.params['maskrad']/pixelsize
if self.params['pixelmaskrad'] > boxsize/2:
apDisplay.printError("Mask radius larger than boxsize")
if self.params['numpart'] is None:
self.params['numpart'] = numpart
if self.params['numpart'] > numpart:
apDisplay.printError("Requested more particles than available in stack")
if self.params['numrefs'] is None:
### num refs should go approx. as the sqrt of the number of particles
self.params['numrefs'] = int(math.ceil(math.sqrt(self.params['numpart'])))+1
def start(self):
"""
this is the main component of the script
where all the processing is done
"""
### initialize some variables
self.runq = None
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
apDisplay.printMsg("Pixel size: %.5f"%(self.apix))
self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
apDisplay.printMsg("Box size: %d"%(self.boxsize))
self.checkResults()
self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
self.numiter = self.getNumberOfIterations()
for i in range(self.numiter):
iternum = i+1
apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green")
self.uploadIteration(iternum)
reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir'])
if reconrunid:
apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid'])
apRecon.setGoodBadParticlesFromReconId(reconrunid)
else:
apDisplay.printWarning("Could not find recon run id")
def runRmeasure(self, volfile):
emancmd = "proc3d %s %s"%(volfile, "rmeasure.mrc")
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin']
self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc")
#apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan")
apFile.removeFile("rmeasure.mrc")
def medianVolume(self):
volpath = os.path.join(self.params['rundir'], "volumes/*a.mrc")
mrcfiles = glob.glob(volpath)
volumes = []
for filename in mrcfiles:
if os.path.isfile(filename):
vol = mrc.read(filename)
print filename, vol.shape
volumes.append(vol)
volarray = numpy.asarray(volumes, dtype=numpy.float32)
try:
medarray = numpy.median(volarray, axis=0)
except:
medarray = numpy.median(volarray)
medfile = os.path.join(self.params['rundir'], "volumes/medianVolume.mrc")
print medfile, medarray.shape
mrc.write(medarray, medfile)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
uploadcmd = ( ("uploadModel.py --projectid=%d --session=%s --file=%s "
+"--apix=%.3f --sym=%s --name=satmedian-recon%d.mrc --res=30 --description='%s %d'")
%(self.params['projectid'], sessiondata['name'], medfile,
apix, self.params['symmname'], self.params['reconid'],
"SAT selected median volume for recon", self.params['reconid'], ) )
apDisplay.printColor(uploadcmd, "purple")
f = open("upload.sh", "w")
f.write(uploadcmd+"\n")
f.close()
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.checkNoRefRun()
### convert stack to spider
spiderstack = self.createSpiderFile()
### create initialization template
if self.params['initmethod'] == 'allaverage':
templatefile = self.averageTemplate()
elif self.params['initmethod'] == 'selectrand':
templatefile = self.selectRandomParticles()
elif self.params['initmethod'] == 'randpart':
templatefile = self.pickRandomParticle()
elif self.params['initmethod'] == 'template':
templatefile = self.getTemplate()
else:
apDisplay.printError("unknown initialization method defined: "
+str(self.params['initmethod'])+" not in "+str(self.initmethods))
apDisplay.printColor("Running spider this can take awhile","cyan")
### run the alignment
aligntime = time.time()
pixrad = int(round(self.params['partrad']/self.stack['apix']/self.params['bin']))
alignedstack, self.partlist = alignment.refFreeAlignParticles(
spiderstack, templatefile,
self.params['numpart'], pixrad,
self.params['firstring'], self.params['lastring'],
rundir = ".")
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=False)
### convert stack to imagic
imagicstack = self.convertSpiderStack(alignedstack)
apFile.removeFile(alignedstack)
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = aligntime
self.insertNoRefRun(imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def setupParticles(self):
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
apix = self.stack['apix']*self.params['bin']
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1:
clipsize = boxsize*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
if self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
proccmd += " last="+str(self.params['numpart']-1)
apFile.removeStack(self.params['localstack'], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
return (boxsize, apix)
def runMaxlike(self):
stackdata = apStack.getOnlyStackData(self.params['stackid'])
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
### process stack to local file
self.params['localstack'] = os.path.join(self.params['rundir'],
self.timestamp + ".hed")
proccmd = "proc2d " + stackfile + " " + self.params[
'localstack'] + " apix=" + str(apix)
if self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -nref 1 " + " -iter 10 " + " -o " +
os.path.join(self.params['rundir'], "part" + self.timestamp) +
" -fast -C 1e-18 ")
### angle step
if self.params['rotate'] is True:
xmippopts += " -psi_step 90 "
else:
xmippopts += " -psi_step 360 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
if self.params['maxshift'] is not None:
xmippopts += " -max_shift %d " % (self.params['maxshift'])
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### create a quick mrc
emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc"
apEMAN.executeEmanCmd(emancmd, verbose=True)
apFile.removeStack(self.params['localstack'])
apFile.removeFilePattern("partfiles/*")
def insertAlignStackRunIntoDatabase(self, alignimagicfile):
apDisplay.printMsg("Inserting CL2D Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.runparams['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
# if uniquerun:
# apDisplay.printError("Run name '"+self.runparams['runname']+"' and path already exist in database")
### setup cl2d run
cl2dq = appiondata.ApCL2DRunData()
cl2dq['runname'] = self.runparams['runname']
cl2dq['run_seconds'] = self.runparams['runtime']
cl2dq['fast'] = self.runparams['fast']
self.cl2dqdata=cl2dq
### finish alignment run
alignrunq['cl2drun'] = cl2dq
alignrunq['hidden'] = False
alignrunq['runname'] = self.runparams['runname']
alignrunq['description'] = self.runparams['description']
alignrunq['lp_filt'] = self.runparams['lowpass']
alignrunq['hp_filt'] = self.runparams['highpass']
alignrunq['bin'] = self.runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
if self.runparams['align'] is True: ### option to create aligned stack
alignstackq['imagicfile'] = alignimagicfile
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
alignstackq['iteration'] = self.runparams['maxiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
if self.runparams['align'] is True: ### option to create aligned stack
alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: "+alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(self.runparams['stackid'])
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
alignstackq['description'] = self.runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.runparams['numpart']
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def getBeamTiltFromStackParticleNumber(self):
stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum+1, msg=True)
pdata = stackparticledata['particle']
### step 1: get axial beam tilt on to image coordinate direction
imageshift = pdata['image']['scope']['image shift']
zerobeamtilt = {'x':0.0e-3, 'y':0.0e-3}
axialbeamtilt = self.transformImageShiftToBeamTilt(imageshift, self.tem, self.cam, self.ht, zerobeamtilt, self.mag)
beamtilt_rotationtransform = self.retrieveBeamTiltMatrix(self.tem, self.cam, self.ht, self.mag)
axialbtvector = numpy.array((axialbeamtilt['y'],axialbeamtilt['x']))
axialbtvector_on_image = numpy.dot(beamtilt_rotationtransform, axialbtvector)
apDisplay.printMsg( "Axial Beam Tilt on Image Coordinate ( %5.2f, %5.2f)" % (axialbtvector_on_image[1]*1e3,axialbtvector_on_image[0]*1e3))
### step 2: get off-axis beam tilt
pcoord = {'x':pdata['xcoord']*self.imgpix,'y':pdata['ycoord']*self.imgpix}
apix = apStack.getStackPixelSizeFromStackId(self.stackid)
camdata = pdata['image']['camera']
cambin = camdata['binning']
camdim = camdata['dimension']
camoff = camdata['offset']
camcenter = {'x':self.camsize['x']/2,'y':self.camsize['y']/2}
apDisplay.printMsg( 'Particle coord (%5d,%5d)' % (pdata['xcoord'],pdata['ycoord']))
mcoord = {'y':pdata['ycoord']*cambin['y']+camoff['y']-camcenter['y'],
'x':pdata['xcoord']*cambin['x']+camoff['x']-camcenter['x']}
apDisplay.printMsg( 'Particle from center in camera pixel (%5d,%5d)' % (mcoord['x'],mcoord['y']))
coord = {'x':mcoord['x']*self.imgpix*cambin['x'],
'y':mcoord['y']*self.imgpix*cambin['y']}
apDisplay.printMsg( 'Particle from center (A) (%5.2f,%5.2f)' % (coord['x']*1e10,coord['y']*1e10))
offaxisbeamtilt = self.getCombinedOffAxisBeamTilt(self.c2diameter,self.beamdiameter,coord)
#offaxisbeamtilt = {'x':0.0,'y':0.0}
### step 3: combine beam tilts
totalbeamtilt = {'x':axialbtvector_on_image[1]+offaxisbeamtilt['x'],
'y':axialbtvector_on_image[0]-offaxisbeamtilt['y'] }
apDisplay.printMsg( "Total Beam Tilt on Image Coordinate (mrad) ( %5.2f, %5.2f)" % (totalbeamtilt['x']*1e3,totalbeamtilt['y']*1e3))
return totalbeamtilt
def start(self):
### load parameters
runparams = self.readRunParameters()
### align references
self.alignReferences(runparams)
### create an aligned stack
self.createAlignedReferenceStack()
### read particles
self.lastiter = self.findLastIterNumber()
if self.params['sort'] is True:
self.sortFolder()
reflist = self.readRefDocFile()
partlist = self.readPartDocFile(reflist)
self.writePartDocFile(partlist)
### create aligned stacks
alignimagicfile = self.createAlignedStacks(partlist, runparams['localstack'])
apStack.averageStack(alignimagicfile)
### calculate resolution for each reference
apix = apStack.getStackPixelSizeFromStackId(runparams['stackid'])*runparams['bin']
self.calcResolution(partlist, alignimagicfile, apix)
### insert into databse
self.insertRunIntoDatabase(alignimagicfile, runparams)
self.insertParticlesIntoDatabase(runparams['stackid'], partlist)
apFile.removeStack(runparams['localstack'], warn=False)
rmcmd = "/bin/rm -fr partfiles/*"
apEMAN.executeEmanCmd(rmcmd, verbose=False, showcmd=False)
apFile.removeFilePattern("partfiles/*")
def setupParticles(self):
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
apix = self.stack['apix']*self.params['bin']
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1:
clipsize = boxsize*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
if self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
proccmd += " last="+str(self.params['numpart']-1)
apFile.removeStack(self.params['localstack'], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
return (boxsize, apix)
def insertAlignStackRunIntoDatabase(self, alignimagicfile):
apDisplay.printMsg("Inserting CL2D Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.runparams['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
# if uniquerun:
# apDisplay.printError("Run name '"+self.runparams['runname']+"' and path already exist in database")
### setup cl2d run
cl2dq = appiondata.ApCL2DRunData()
cl2dq['runname'] = self.runparams['runname']
cl2dq['run_seconds'] = self.runparams['runtime']
cl2dq['fast'] = self.runparams['fast']
self.cl2dqdata=cl2dq
### finish alignment run
alignrunq['cl2drun'] = cl2dq
alignrunq['hidden'] = False
alignrunq['runname'] = self.runparams['runname']
alignrunq['description'] = self.runparams['description']
alignrunq['lp_filt'] = self.runparams['lowpass']
alignrunq['hp_filt'] = self.runparams['highpass']
alignrunq['bin'] = self.runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
if self.runparams['align'] is True: ### option to create aligned stack
alignstackq['imagicfile'] = alignimagicfile
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
alignstackq['iteration'] = self.runparams['maxiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
if self.runparams['align'] is True: ### option to create aligned stack
alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: "+alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(self.runparams['stackid'])
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
alignstackq['description'] = self.runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.runparams['numpart']
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def convertStackToSpider(self, emanstackfile, classnum):
"""
takes the stack file and creates a spider file ready for processing
"""
if not os.path.isfile(emanstackfile):
apDisplay.printError("stackfile does not exist: "+emanstackfile)
### first high pass filter particles
apDisplay.printMsg("pre-filtering particles")
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
emancmd = ("proc2d "+emanstackfile+" "+emanstackfile
+" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
+" inplace")
apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc2d "
emancmd += emanstackfile+" "
spiderstack = os.path.join(self.params['rundir'], str(classnum), "otrstack"+self.timestamp+".spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
return spiderstack
def start(self):
"""
this is the main component of the script
where all the processing is done
"""
### initialize some variables
self.runq = None
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
apDisplay.printMsg("Pixel size: %.5f"%(self.apix))
self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
apDisplay.printMsg("Box size: %d"%(self.boxsize))
self.checkResults()
self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
self.numiter = self.getNumberOfIterations()
for i in range(self.numiter):
iternum = i+1
apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green")
self.uploadIteration(iternum)
reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir'])
if reconrunid:
apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid'])
apRecon.setGoodBadParticlesFromReconId(reconrunid)
else:
apDisplay.printWarning("Could not find recon run id")
def runEoTest(self, corrSelectOdd, corrSelectEven, cnum, apshstack, apsheuler, iternum):
apshOddVolfile = os.path.join(self.params['rundir'], str(cnum), "apshVolume_Odd-%03d.spi"%(iternum))
apshEvenVolfile = os.path.join(self.params['rundir'], str(cnum), "apshVolume_Even-%03d.spi"%(iternum))
self.APSHbackProject(apshstack, apsheuler, apshOddVolfile, cnum, corrSelectOdd)
self.APSHbackProject(apshstack, apsheuler, apshEvenVolfile, cnum, corrSelectEven)
fscout = os.path.join(self.params['rundir'], str(cnum), "FSCout-%03d.spi"%(iternum))
backproject.calcFSC(apshOddVolfile, apshEvenVolfile, fscout)
### Calculate FSC - taken from Neil's RCT script
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin']
emancmd = "proc3d %s %s"%(apshEvenVolfile, apshEvenVolfile+".mrc")
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
emancmd = "proc3d %s %s"%(apshOddVolfile, apshOddVolfile+".mrc")
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
fscfile = os.path.join(self.params['rundir'], "fscdata%s.fsc"%(self.timestamp))
emancmd = "proc3d %s %s fsc=%s"%(apshEvenVolfile+".mrc", apshOddVolfile+".mrc", fscfile)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
if not os.path.isfile(fscfile):
apDisplay.printError("Even-Odd fsc calculation failed")
boxsize = self.getBoxSize()
self.fscresolution = apRecon.getResolutionFromFSCFile(fscfile, boxsize, apix, msg=True)
apDisplay.printColor( ("Final FSC resolution: %.5f" % (self.fscresolution)), "cyan")
return fscout
def checkConflicts(self):
if self.params['runname'] is None:
apDisplay.printError("enter a stack run name, e.g. combinestack1")
if self.params['description'] is None:
apDisplay.printError("enter a stack description")
if self.params['stacks'] and ',' in self.params['stacks']:
#remember stackids are a list of strings
stackids = self.params['stacks'].split(',')
self.params['stackids'] = stackids
else:
apDisplay.printError("enter a list of stack ids to combine, e.g. --stackids=11,14,7")
### check to make sure all pixel and box size are the same
self.newboxsize = None
self.newpixelsize = None
for stackidstr in self.params['stackids']:
if not re.match("^[0-9]+$", stackidstr):
apDisplay.printError("Stack id '%s' is not an integer"%(stackidstr))
stackid = int(stackidstr)
boxsize = apStack.getStackBoxsize(stackid, msg=False)
pixelsize = apStack.getStackPixelSizeFromStackId(stackid, msg=False)
apDisplay.printMsg("Stack id: %d\tBoxsize: %d\tPixelsize: %.3f"%(stackid, boxsize, pixelsize))
if self.newboxsize is None:
self.newboxsize = boxsize
if self.newpixelsize is None:
self.newpixelsize = pixelsize
if boxsize != self.newboxsize:
apDisplay.printError("Trying to combine stacks with different box sizes")
if abs(pixelsize - self.newpixelsize) > 0.01:
apDisplay.printError("Trying to combine stacks with different pixel sizes")
def runMaxlike(self):
stackdata = apStack.getOnlyStackData(self.params["stackid"])
apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"])
### process stack to local file
self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed")
proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(apix)
if self.params["highpass"] > 1:
proccmd += " hp=" + str(self.params["highpass"])
if self.params["lowpass"] > 1:
proccmd += " lp=" + str(self.params["lowpass"])
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" "
+ " -i "
+ os.path.join(self.params["rundir"], self.partlistdocfile)
+ " -nref 1 "
+ " -iter 10 "
+ " -o "
+ os.path.join(self.params["rundir"], "part" + self.timestamp)
+ " -fast -C 1e-18 "
)
### angle step
if self.params["rotate"] is True:
xmippopts += " -psi_step 90 "
else:
xmippopts += " -psi_step 360 "
### convergence criteria
if self.params["converge"] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params["converge"] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params["mirror"] is True:
xmippopts += " -mirror "
if self.params["maxshift"] is not None:
xmippopts += " -max_shift %d " % (self.params["maxshift"])
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime))
### create a quick mrc
emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc"
apEMAN.executeEmanCmd(emancmd, verbose=True)
apFile.removeStack(self.params["localstack"])
apFile.removeFilePattern("partfiles/*")
def start(self):
### load parameters
self.runparams = self.readRunParameters()
self.apix = apStack.getStackPixelSizeFromStackId(
self.runparams['stackid']) * self.runparams['bin']
self.Nlevels = len(
glob.glob("part" + self.params['timestamp'] + "_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part" + self.params[
'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
if self.runparams['align']:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels - 1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
if self.runparams['align']:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part" + self.params['timestamp'] +
"_level_%02d_.hed" % level, classnum + 1,
partdict[classnum], len(partdict))
def start(self):
"""
This is the core of your function.
"""
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp+".hed")
apFile.removeStack(filtered_stackfile, warn=False)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
boxsize = apStack.getStackBoxsize(self.params['stackid'])
emancmd = "proc2d %s %s apix=%.3f "%(original_stackfile, filtered_stackfile, apix)
if self.params['lowpass'] is not None:
emancmd += " lp=%.3f "%(self.params['lowpass'])
if self.params['highpass'] is not None:
emancmd += " hp=%.3f "%(self.params['highpass'])
if self.params['bin'] is not None and self.params['bin'] > 1:
## determine a multiple of the bin that is divisible by 2 and less than the boxsize
clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2*self.params['bin']
emancmd += " shrink=%d clip=%d,%d "%(self.params['bin'], clipsize, clipsize)
emancmd += " last=%d "%(self.params['numpart']-1)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
### confirm that it worked
if self.params['numpart'] != apFile.numImagesInStack(filtered_stackfile):
apDisplay.printError("Missing particles in stack")
### run the radon transform code
self.radonAlign(filtered_stackfile)
### insert info into database
self.commitToDatabase()
def runRmeasure(self):
finalrawvolfile = os.path.join(self.params['rundir'], "rawvolume%s-%03d.spi"%(self.timestamp, self.params['numiters']))
emancmd = "proc3d %s %s"%(finalrawvolfile, "rmeasure.mrc")
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin']
self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc")
#apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan")
apFile.removeFile("rmeasure.mrc")
def insertOtrRun(self, volfile):
### setup resolutions
fscresq = appiondata.ApResolutionData()
fscresq["type"] = "fsc"
fscresq["half"] = self.fscresolution
fscresq["fscfile"] = "fscdata" + self.timestamp + ".fsc"
rmeasureq = appiondata.ApResolutionData()
rmeasureq["type"] = "rmeasure"
rmeasureq["half"] = self.rmeasureresolution
rmeasureq["fscfile"] = None
### insert rct run data
otrrunq = appiondata.ApOtrRunData()
otrrunq["runname"] = self.params["runname"]
tempstr = ""
for cnum in self.classlist:
tempstr += str(cnum) + ","
classliststr = tempstr[:-1]
otrrunq["classnums"] = classliststr
otrrunq["numiter"] = self.params["numiters"]
otrrunq["euleriter"] = self.params["refineiters"]
otrrunq["maskrad"] = self.params["radius"]
otrrunq["lowpassvol"] = self.params["lowpassvol"]
otrrunq["highpasspart"] = self.params["highpasspart"]
otrrunq["median"] = self.params["median"]
otrrunq["description"] = self.params["description"]
otrrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
otrrunq["alignstack"] = self.alignstackdata
otrrunq["tiltstack"] = apStack.getOnlyStackData(self.params["tiltstackid"])
otrrunq["numpart"] = self.numpart
otrrunq["fsc_resolution"] = fscresq
otrrunq["rmeasure_resolution"] = rmeasureq
if self.params["commit"] is True:
otrrunq.insert()
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq["otrrun"] = otrrunq
densq["path"] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile)))
densq["name"] = os.path.basename(volfile)
densq["hidden"] = False
densq["norm"] = True
densq["symmetry"] = appiondata.ApSymmetryData.direct_query(25)
densq["pixelsize"] = apStack.getStackPixelSizeFromStackId(self.params["partstackid"]) * self.params["bin"]
densq["boxsize"] = self.getBoxSize()
densq["lowpass"] = self.params["lowpassvol"]
densq["highpass"] = self.params["highpasspart"]
densq["mask"] = self.params["radius"]
# densq['iterid'] = self.params['numiters']
densq["description"] = self.params["description"]
densq["resolution"] = self.fscresolution
densq["rmeasure"] = self.rmeasureresolution
densq["session"] = apStack.getSessionDataFromStackId(self.params["tiltstackid"])
densq["md5sum"] = apFile.md5sumfile(volfile)
if self.params["commit"] is True:
densq.insert()
return
def insertOtrRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq['type'] = "fsc" fscresq['half'] = self.fscresolution fscresq['fscfile'] = "fscdata"+self.timestamp+".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq['type'] = "rmeasure" rmeasureq['half'] = self.rmeasureresolution rmeasureq['fscfile'] = None ### insert rct run data otrrunq = appiondata.ApOtrRunData() otrrunq['runname'] = self.params['runname'] tempstr = "" for cnum in self.classlist: tempstr += str(cnum)+"," classliststr = tempstr[:-1] otrrunq['classnums'] = classliststr otrrunq['numiter'] = self.params['numiters'] otrrunq['euleriter'] = self.params['refineiters'] otrrunq['maskrad'] = self.params['radius'] otrrunq['lowpassvol'] = self.params['lowpassvol'] otrrunq['highpasspart'] = self.params['highpasspart'] otrrunq['median'] = self.params['median'] otrrunq['description'] = self.params['description'] otrrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) otrrunq['alignstack'] = self.alignstackdata otrrunq['tiltstack'] = apStack.getOnlyStackData(self.params['tiltstackid']) otrrunq['numpart'] = self.numpart otrrunq['fsc_resolution'] = fscresq otrrunq['rmeasure_resolution'] = rmeasureq if self.params['commit'] is True: otrrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['otrrun'] = otrrunq densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25) densq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] densq['boxsize'] = self.getBoxSize() densq['lowpass'] = self.params['lowpassvol'] densq['highpass'] = self.params['highpasspart'] densq['mask'] = self.params['radius'] #densq['iterid'] = self.params['numiters'] densq['description'] = self.params['description'] densq['resolution'] = self.fscresolution densq['rmeasure'] = self.rmeasureresolution densq['session'] = apStack.getSessionDataFromStackId(self.params['tiltstackid']) densq['md5sum'] = apFile.md5sumfile(volfile) if self.params['commit'] is True: densq.insert() return
def insertRunIntoDatabase(self, alignedPartStack, alignedClassStack, runparams):
apDisplay.printMsg("Inserting ISAC Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = runparams['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup ISAC like run
isacq = appiondata.ApSparxISACRunData()
isacq['runname'] = runparams['runname']
isacq['job'] = self.getISACJobData(runparams)
### finish alignment run
alignrunq['isacrun'] = isacq
alignrunq['hidden'] = False
alignrunq['runname'] = runparams['runname']
alignrunq['description'] = runparams['description']
alignrunq['lp_filt'] = runparams['lowpass']
alignrunq['hp_filt'] = runparams['highpass']
alignrunq['bin'] = runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = alignedPartStack
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = alignedClassStack
alignstackq['iteration'] = self.lastiter
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: "+alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(runparams['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])*self.params['bin']
alignstackq['description'] = runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.numPart
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def getBeamTiltFromStackParticleNumber(self):
stackparticledata = apStack.getOneParticleFromStackId(self.stackid,
self.imgnum + 1,
msg=True)
pdata = stackparticledata['particle']
### step 1: get axial beam tilt on to image coordinate direction
imageshift = pdata['image']['scope']['image shift']
zerobeamtilt = {'x': 0.0e-3, 'y': 0.0e-3}
axialbeamtilt = self.transformImageShiftToBeamTilt(
imageshift, self.tem, self.cam, self.ht, zerobeamtilt, self.mag)
beamtilt_rotationtransform = self.retrieveBeamTiltMatrix(
self.tem, self.cam, self.ht, self.mag)
axialbtvector = numpy.array((axialbeamtilt['y'], axialbeamtilt['x']))
axialbtvector_on_image = numpy.dot(beamtilt_rotationtransform,
axialbtvector)
apDisplay.printMsg(
"Axial Beam Tilt on Image Coordinate ( %5.2f, %5.2f)" %
(axialbtvector_on_image[1] * 1e3, axialbtvector_on_image[0] * 1e3))
### step 2: get off-axis beam tilt
pcoord = {
'x': pdata['xcoord'] * self.imgpix,
'y': pdata['ycoord'] * self.imgpix
}
apix = apStack.getStackPixelSizeFromStackId(self.stackid)
camdata = pdata['image']['camera']
cambin = camdata['binning']
camdim = camdata['dimension']
camoff = camdata['offset']
camcenter = {'x': self.camsize['x'] / 2, 'y': self.camsize['y'] / 2}
apDisplay.printMsg('Particle coord (%5d,%5d)' %
(pdata['xcoord'], pdata['ycoord']))
mcoord = {
'y': pdata['ycoord'] * cambin['y'] + camoff['y'] - camcenter['y'],
'x': pdata['xcoord'] * cambin['x'] + camoff['x'] - camcenter['x']
}
apDisplay.printMsg('Particle from center in camera pixel (%5d,%5d)' %
(mcoord['x'], mcoord['y']))
coord = {
'x': mcoord['x'] * self.imgpix * cambin['x'],
'y': mcoord['y'] * self.imgpix * cambin['y']
}
apDisplay.printMsg('Particle from center (A) (%5.2f,%5.2f)' %
(coord['x'] * 1e10, coord['y'] * 1e10))
offaxisbeamtilt = self.getCombinedOffAxisBeamTilt(
self.c2diameter, self.beamdiameter, coord)
#offaxisbeamtilt = {'x':0.0,'y':0.0}
### step 3: combine beam tilts
totalbeamtilt = {
'x': axialbtvector_on_image[1] + offaxisbeamtilt['x'],
'y': axialbtvector_on_image[0] - offaxisbeamtilt['y']
}
apDisplay.printMsg(
"Total Beam Tilt on Image Coordinate (mrad) ( %5.2f, %5.2f)" %
(totalbeamtilt['x'] * 1e3, totalbeamtilt['y'] * 1e3))
return totalbeamtilt
def checkConflicts(self):
if self.params["stackid"] is None:
apDisplay.printError("Stack id was not defined")
if self.params["mandir"] is None:
apDisplay.printError("Directory containing mandatory files was not defined")
if self.params["description"] is None:
apDisplay.printError("Run description was not defined")
stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False)
stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"])
if self.params["numpart"] is None:
self.params["numpart"] = apFile.numImagesInStack(stackfile)
apDisplay.printWarning("Number of particles not defined, processing full stack")
if self.params["numpart"] > apFile.numImagesInStack(stackfile):
apDisplay.printError(
"trying to use more particles "
+ str(self.params["numpart"])
+ " than available "
+ str(apFile.numImagesInStack(stackfile))
)
if self.params["replen"] is None:
apDisplay.printError("Helical repeat was not defined")
if self.params["subunits"] is None:
apDisplay.printError("Number of subunits was not defined")
if self.params["diameter"] is None:
apDisplay.printError("Filament diameter was not defined")
if self.params["diaminner"] is None:
apDisplay.printError("Inner filament diameter was not defined")
if self.params["rescut"] is None:
apDisplay.printError("Phase residual cutoff was not defined")
if self.params["filval"] is None:
apDisplay.printError("Filter value was not defined")
if self.params["bin"] is None:
self.params["bin"] = 1
apDisplay.printWarning("Binning factor was not defined, binning by one")
if self.params["cont"] is None:
apDisplay.printError("Must specify if a contrast change is needed")
if self.params["prehip"] is None:
apDisplay.printError("Must specify if prehip needs to be executed or not")
step = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
self.params["step"] = step
onerep = math.floor(self.params["replen"] / step)
calcyht2 = math.floor(self.params["diameter"] / step)
boxsize = apStack.getStackBoxsize(self.params["stackid"])
self.params["boxsize"] = boxsize
if self.params["xlngth"] < onerep:
apDisplay.printError("Filament segment length can not be less than one helical repeat")
if self.params["xlngth"] > boxsize:
apDisplay.printError("Filament segment length can not be greater than stack box size")
if self.params["yht2"] < calcyht2:
apDisplay.printError("Filament box height can not be less than filament diameter")
if self.params["padval"] < self.params["xlngth"]:
apDisplay.printError("Pad value can not be less than the filament segment length")
if not self.checkval(self.params["yht2"]):
apDisplay.printError("Filament box height must be a power of two")
if not self.checkval(self.params["padval"]):
apDisplay.printError("Pad value must be a power of two")
def __initializeStack(self):
# This sets parameters of our new stack based on the params of the original stack
# TODO: use originalStackData to populate these
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.stack['format'] = 'imagic'
self.stack['phaseflipped'] = self.originalStackData.phaseFlipped
def processVolume(self, spivolfile, iternum=0):
### set values
apix = apStack.getStackPixelSizeFromStackId(
self.params['tiltstackid']) * self.params['tiltbin']
boxsize = self.getBoxSize()
rawspifile = os.path.join(
self.params['rundir'],
"rawvolume%s-%03d.spi" % (self.timestamp, iternum))
mrcvolfile = os.path.join(
self.params['rundir'],
"volume%s-%03d.mrc" % (self.timestamp, iternum))
lowpass = self.params['lowpassvol']
### copy original to raw file
shutil.copy(spivolfile, rawspifile)
### convert to mrc
emancmd = ("proc3d " + spivolfile + " " + mrcvolfile +
" norm=0,1 apix=" + str(apix))
apEMAN.executeEmanCmd(emancmd, verbose=False)
### median filter
rawvol = mrc.read(mrcvolfile)
medvol = ndimage.median_filter(rawvol, size=self.params['median'])
mrc.write(medvol, mrcvolfile)
### low pass filter
emancmd = ("proc3d " + mrcvolfile + " " + mrcvolfile +
" center norm=0,1 apix=" + str(apix) + " lp=" +
str(lowpass))
apEMAN.executeEmanCmd(emancmd, verbose=False)
### set origin
emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " origin=0,0,0 "
apEMAN.executeEmanCmd(emancmd, verbose=False)
### mask volume
emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " mask=" + str(
self.params['radius'])
apEMAN.executeEmanCmd(emancmd, verbose=False)
### convert to spider
apFile.removeFile(spivolfile)
emancmd = "proc3d " + mrcvolfile + " " + spivolfile + " spidersingle"
apEMAN.executeEmanCmd(emancmd, verbose=False)
### image with chimera
if self.params['skipchimera'] is False:
if self.params['mass'] is not None:
apDisplay.printMsg("Using scale by mass method")
apChimera.setVolumeMass(mrcvolfile,
apix=apix,
mass=self.params['mass'])
apChimera.renderSnapshots(mrcvolfile, self.params['contour'],
self.params['zoom'], 'c1')
return mrcvolfile
def processVolume(self, spivolfile, cnum, iternum=0):
### set values
apix = apStack.getStackPixelSizeFromStackId(
self.params['tiltstackid']) * self.params['bin']
boxsize = self.getBoxSize()
volfilename = os.path.splitext(spivolfile)[0]
rawspifile = volfilename + "-raw.spi"
mrcvolfile = volfilename + ".mrc"
lowpass = self.params['lowpassvol']
### copy original to raw file
shutil.copy(spivolfile, rawspifile)
### convert to mrc
emancmd = ("proc3d " + spivolfile + " " + mrcvolfile +
" norm=0,1 apix=" + str(apix))
apEMAN.executeEmanCmd(emancmd, verbose=False)
### median filter
rawvol = mrc.read(mrcvolfile)
medvol = ndimage.median_filter(rawvol, size=self.params['median'])
mrc.write(medvol, mrcvolfile)
### low pass filter
emancmd = ("proc3d " + mrcvolfile + " " + mrcvolfile +
" center norm=0,1 apix=" + str(apix) + " lp=" +
str(lowpass))
apEMAN.executeEmanCmd(emancmd, verbose=False)
### set origin
emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " origin=0,0,0 "
apEMAN.executeEmanCmd(emancmd, verbose=False)
### mask volume
emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " mask=" + str(
self.params['radius'])
apEMAN.executeEmanCmd(emancmd, verbose=False)
### convert to spider
apFile.removeFile(spivolfile)
emancmd = "proc3d " + mrcvolfile + " " + spivolfile + " spidersingle"
apEMAN.executeEmanCmd(emancmd, verbose=False)
### image with chimera
if self.params['skipchimera'] is False:
apChimera.renderSnapshots(mrcvolfile, self.params['contour'],
self.params['zoom'], 'c1')
animationthread = threading.Thread(
target=apChimera.renderAnimation,
args=(mrcvolfile, self.params['contour'], self.params['zoom'],
'c1'))
animationthread.setDaemon(1)
animationthread.start()
return mrcvolfile
def runRmeasure(self):
finalrawvolfile = os.path.join(
self.params['rundir'],
"rawvolume%s-%03d.spi" % (self.timestamp, self.params['numiters']))
emancmd = "proc3d %s %s" % (finalrawvolfile, "rmeasure.mrc")
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
apix = apStack.getStackPixelSizeFromStackId(
self.params['tiltstackid']) * self.params['tiltbin']
self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc")
#apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan")
apFile.removeFile("rmeasure.mrc")
def __initializeStack(self):
# This sets parameters of our new stack based on the params of the original stack
# TODO: use originalStackData to populate these
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.stack['format'] = 'imagic'
self.stack['phaseflipped'] = self.originalStackData.phaseFlipped
def convertStackToSpider(self, emanstackfile):
"""
takes the stack file and creates a spider file ready for processing
"""
if not os.path.isfile(emanstackfile):
apDisplay.printError("stackfile does not exist: " + emanstackfile)
tempstack = os.path.join(self.params['rundir'],
"filter" + self.timestamp + ".hed")
### first high pass filter particles
apDisplay.printMsg("pre-filtering particles")
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
boxsize = self.getBoxSize()
emancmd = ("proc2d " + emanstackfile + " " + tempstack + " apix=" +
str(apix) + " ")
if self.params['highpasspart'] is not None and self.params[
'highpasspart'] > 0:
emancmd += "hp=" + str(self.params['highpasspart']) + " "
if self.params[
'lowpasspart'] is not None and self.params['lowpasspart'] > 0:
emancmd += "lp=" + str(self.params['lowpasspart']) + " "
if self.params['tiltbin'] > 1:
clipsize = boxsize * self.params['tiltbin']
emancmd += " shrink=%d clip=%d,%d " % (self.params['tiltbin'],
clipsize, clipsize)
apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc2d "
emancmd += tempstack + " "
spiderstack = os.path.join(self.params['rundir'],
"rctstack" + self.timestamp + ".spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack + " "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor(
"Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
time.sleep(1) # wait a sec, for things to finish
apDisplay.printColor(
"finished eman in " +
apDisplay.timeString(time.time() - starttime), "cyan")
apFile.removeStack(tempstack, warn=False)
apFile.removeStack(emanstackfile, warn=False)
if not os.path.isfile(spiderstack):
apDisplay.printError("Failed to create a spider stack")
return spiderstack
def convertVolToSpider(self,
mrcvolfile=None,
modelid=None,
apix=None,
spivolfile=None):
"""
takes the mrc volume and creates a spider file ready for processing
"""
if modelid is not None:
initModelData = appiondata.ApInitialModelData.direct_query(modelid)
mrcvolfile = initModelData['path']['path'] + "/" + initModelData[
'name']
apix = initModelData['pixelsize']
stackapix = apStack.getStackPixelSizeFromStackId(
self.params['partstackid']) * self.params['bin']
stackboxsize = apStack.getStackBoxsize(
self.params['partstackid']) / self.params['bin']
print apix
print mrcvolfile
if not os.path.isfile(mrcvolfile):
apDisplay.printError("volfile does not exist: " + mrcvolfile)
### first high pass filter particles
#apDisplay.printMsg("pre-filtering particles")
#apix = apStack.getStackPixelSizeFromStackId(self.params['partstackid'])
#emancmd = ("proc3d "+mrcvolfile+" "+emanstackfile
# +" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
# +" inplace")
#apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc3d "
emancmd += mrcvolfile + " "
if spivolfile is None:
spivolfile = os.path.join(self.params['rundir'], "threed-0a.spi")
apFile.removeFile(spivolfile, warn=True)
emancmd += spivolfile + " "
emancmd += "scale=" + str(apix / stackapix) + " "
emancmd += "clip=" + str(stackboxsize) + "," + str(
stackboxsize) + "," + str(stackboxsize) + " "
emancmd += "spidersingle"
starttime = time.time()
apDisplay.printColor("Running spider volume conversion", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor(
"finished conversion in " +
apDisplay.timeString(time.time() - starttime), "cyan")
return spivolfile
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
### get stack data
self.stack = {}
if self.params['stackid'] is not None:
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
else:
self.stack['data'] = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
self.stack['apix'] = self.stack['pixelsize']
self.stack['boxsize'] = self.stack['boxsize']
self.stack['file'] = self.stack['imagicfile']
### check conflicts
if self.params['stackid'] is None and self.params['alignstackid'] is None:
apDisplay.printError("stack id OR alignstack id was not defined")
if self.params['stackid'] is not None and self.params['alignstackid'] is not None:
apDisplay.printError("either specify stack id OR alignstack id, not both")
if self.params['generations'] is None:
apDisplay.printError("number of generations was not provided")
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
if self.params['numpart'] > apFile.numImagesInStack(self.stack['file']):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(self.stack['file'])))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(self.stack['file'])
boxsize = apStack.getStackBoxsize(self.params['stackid'])
if self.params['ou'] is None:
self.params['ou'] = (boxsize / 2.0) - 2
self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
self.mpirun = self.checkMPI()
if self.mpirun is None:
apDisplay.printError("There is no MPI installed")
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("Stack id was not defined")
if self.params['mandir'] is None:
apDisplay.printError("Directory containing mandatory files was not defined")
if self.params['description'] is None:
apDisplay.printError("Run description was not defined")
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
apDisplay.printWarning("Number of particles not defined, processing full stack")
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])+" than available "+str(apFile.numImagesInStack(stackfile)))
if self.params['replen'] is None:
apDisplay.printError("Helical repeat was not defined")
if self.params['subunits'] is None:
apDisplay.printError("Number of subunits was not defined")
if self.params['diameter'] is None:
apDisplay.printError("Filament diameter was not defined")
if self.params['diaminner'] is None:
apDisplay.printError("Inner filament diameter was not defined")
if self.params['rescut'] is None:
apDisplay.printError("Phase residual cutoff was not defined")
if self.params['filval'] is None:
apDisplay.printError("Filter value was not defined")
if self.params['bin'] is None:
self.params['bin'] = 1
apDisplay.printWarning("Binning factor was not defined, binning by one")
if self.params['cont'] is None:
apDisplay.printError("Must specify if a contrast change is needed")
if self.params['prehip'] is None:
apDisplay.printError("Must specify if prehip needs to be executed or not")
step = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.params['step'] = step
onerep = math.floor(self.params['replen'] / step)
calcyht2 = math.floor(self.params['diameter'] / step)
boxsize = apStack.getStackBoxsize(self.params['stackid'])
self.params['boxsize'] = boxsize
if self.params['xlngth'] < onerep:
apDisplay.printError("Filament segment length can not be less than one helical repeat")
if self.params['xlngth'] > boxsize:
apDisplay.printError("Filament segment length can not be greater than stack box size")
if self.params['yht2'] < calcyht2:
apDisplay.printError("Filament box height can not be less than filament diameter")
if self.params['padval'] < self.params['xlngth']:
apDisplay.printError("Pad value can not be less than the filament segment length")
if not self.checkval(self.params['yht2']):
apDisplay.printError("Filament box height must be a power of two")
if not self.checkval(self.params['padval']):
apDisplay.printError("Pad value must be a power of two")
def useStackForTemplate(self):
apDisplay.printMsg("Using stack to make templates")
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
self.params['sessionname'] = sessiondata['name']
self.params['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
stackdata = apStack.getOnlyStackData(self.params['stackid'])
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
self.params['templatelist'] = []
if self.params['avgstack'] is True:
### averaging all particles together
apDisplay.printMsg("Averaging all images in stack")
templatename = "stack%d-average.mrc" % (self.params['stackid'])
oldtemplatepath = os.path.join(stackdata['path']['path'],
"average.mrc")
abstemplatepath = os.path.join(stackdata['path']['path'],
templatename)
if os.path.isfile(oldtemplatepath):
shutil.copy(oldtemplatepath, abstemplatepath)
else:
### average all images using proc2d
emancmd = "proc2d %s %s average" % (stackfile, abstemplatepath)
apEMAN.executeEmanCmd(emancmd)
if os.path.isfile(abstemplatepath):
self.params['templatelist'] = [abstemplatepath]
elif self.params['imgnums'] is not None:
### check to see if stackimagenum is within the boundary of the stack
numpart = apFile.numImagesInStack(stackfile)
for i in self.params['imgnums'].split(","):
partnum = int(i)
if partnum > numpart or partnum < 0:
apDisplay.printError(
"'imgnums' is NOT within the boundary of the stack: %d > %d"
% (partnum, numpart))
apDisplay.printMsg("Extracting image %d from stack" % partnum)
templatename = "stack%d-particle%d.mrc" % (
self.params['stackid'], partnum)
abstemplatepath = os.path.join(self.params['rundir'],
templatename)
### run proc2d with params['stackimgnum']
emancmd = "proc2d %s %s first=%d last=%d" % (
stackfile, abstemplatepath, partnum, partnum)
### create template
apDisplay.printMsg("creating " + templatename)
apEMAN.executeEmanCmd(emancmd)
if os.path.isfile(abstemplatepath):
self.params['templatelist'].append(abstemplatepath)
def processVolume(self, spivolfile, cnum, iternum=0):
### set values
apix = apStack.getStackPixelSizeFromStackId(self.params["tiltstackid"]) * self.params["bin"]
boxsize = self.getBoxSize()
volfilename = os.path.splitext(spivolfile)[0]
rawspifile = volfilename + "-raw.spi"
mrcvolfile = volfilename + ".mrc"
lowpass = self.params["lowpassvol"]
### copy original to raw file
shutil.copy(spivolfile, rawspifile)
### convert to mrc
emancmd = "proc3d " + spivolfile + " " + mrcvolfile + " norm=0,1 apix=" + str(apix)
apEMAN.executeEmanCmd(emancmd, verbose=False)
### median filter
rawvol = mrc.read(mrcvolfile)
medvol = ndimage.median_filter(rawvol, size=self.params["median"])
mrc.write(medvol, mrcvolfile)
### low pass filter
emancmd = (
"proc3d " + mrcvolfile + " " + mrcvolfile + " center norm=0,1 apix=" + str(apix) + " lp=" + str(lowpass)
)
apEMAN.executeEmanCmd(emancmd, verbose=False)
### set origin
emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " origin=0,0,0 "
apEMAN.executeEmanCmd(emancmd, verbose=False)
### mask volume
emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " mask=" + str(self.params["radius"])
apEMAN.executeEmanCmd(emancmd, verbose=False)
### convert to spider
apFile.removeFile(spivolfile)
emancmd = "proc3d " + mrcvolfile + " " + spivolfile + " spidersingle"
apEMAN.executeEmanCmd(emancmd, verbose=False)
### image with chimera
if self.params["skipchimera"] is False:
apChimera.renderSnapshots(mrcvolfile, self.params["contour"], self.params["zoom"], "c1")
animationthread = threading.Thread(
target=apChimera.renderAnimation, args=(mrcvolfile, self.params["contour"], self.params["zoom"], "c1")
)
animationthread.setDaemon(1)
animationthread.start()
return mrcvolfile
def getInstrumentParams(self):
self.stackpix = apStack.getStackPixelSizeFromStackId(self.stackid, True) * 1e-10
stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum+1, msg=True)
pdata = stackparticledata['particle']
self.imgpix = apDatabase.getPixelSize(pdata['image'])*1e-10
scopedata = pdata['image']['scope']
self.tem = scopedata['tem']
self.cam = pdata['image']['camera']['ccdcamera']
self.ht = scopedata['high tension']
self.Cs = 2e-3
self.mag = scopedata['magnification']
self.camsize = {'x':4096,'y':4096}
self.beamdiameter = 1e-6
self.c2diameter = 1.0e-4
self.wavelength = fftfun.getElectronWavelength(self.ht)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['runname'] is None:
apDisplay.printError("new runname was not defined")
self.params['totalpart'] = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if 'last' not in self.params.keys() or self.params['last'] is None:
self.params['last'] = self.params['totalpart']
self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.refineboxsize = self.boxsize * self.params['bin']
### set cs value
self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
self.modelids = map((lambda x: int(x)),self.params['modelid'].split(','))
self.checkPackageConflicts()
def getInstrumentParams(self):
self.stackpix = apStack.getStackPixelSizeFromStackId(
self.stackid, True) * 1e-10
stackparticledata = apStack.getOneParticleFromStackId(self.stackid,
self.imgnum + 1,
msg=True)
pdata = stackparticledata['particle']
self.imgpix = apDatabase.getPixelSize(pdata['image']) * 1e-10
scopedata = pdata['image']['scope']
self.tem = scopedata['tem']
self.cam = pdata['image']['camera']['ccdcamera']
self.ht = scopedata['high tension']
self.Cs = 2e-3
self.mag = scopedata['magnification']
self.camsize = {'x': 4096, 'y': 4096}
self.beamdiameter = 1e-6
self.c2diameter = 1.0e-4
self.wavelength = fftfun.getElectronWavelength(self.ht)
def processVolume(self, spivolfile, iternum=0):
### set values
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin']
boxsize = self.getBoxSize()
rawspifile = os.path.join(self.params['rundir'], "rawvolume%s-%03d.spi"%(self.timestamp, iternum))
mrcvolfile = os.path.join(self.params['rundir'], "volume%s-%03d.mrc"%(self.timestamp, iternum))
lowpass = self.params['lowpassvol']
### copy original to raw file
shutil.copy(spivolfile, rawspifile)
### convert to mrc
emancmd = ("proc3d "+spivolfile+" "+mrcvolfile+" norm=0,1 apix="+str(apix))
apEMAN.executeEmanCmd(emancmd, verbose=False)
### median filter
rawvol = mrc.read(mrcvolfile)
medvol = ndimage.median_filter(rawvol, size=self.params['median'])
mrc.write(medvol, mrcvolfile)
### low pass filter
emancmd = ("proc3d "+mrcvolfile+" "+mrcvolfile+" center norm=0,1 apix="
+str(apix)+" lp="+str(lowpass))
apEMAN.executeEmanCmd(emancmd, verbose=False)
### set origin
emancmd = "proc3d "+mrcvolfile+" "+mrcvolfile+" origin=0,0,0 "
apEMAN.executeEmanCmd(emancmd, verbose=False)
### mask volume
emancmd = "proc3d "+mrcvolfile+" "+mrcvolfile+" mask="+str(self.params['radius'])
apEMAN.executeEmanCmd(emancmd, verbose=False)
### convert to spider
apFile.removeFile(spivolfile)
emancmd = "proc3d "+mrcvolfile+" "+spivolfile+" spidersingle"
apEMAN.executeEmanCmd(emancmd, verbose=False)
### image with chimera
if self.params['skipchimera'] is False:
if self.params['mass'] is not None:
apDisplay.printMsg("Using scale by mass method")
apChimera.setVolumeMass(mrcvolfile, apix=apix, mass=self.params['mass'])
apChimera.renderSnapshots(mrcvolfile, self.params['contour'], self.params['zoom'], 'c1')
return mrcvolfile
def convertStackToSpider(self, emanstackfile):
"""
takes the stack file and creates a spider file ready for processing
"""
if not os.path.isfile(emanstackfile):
apDisplay.printError("stackfile does not exist: "+emanstackfile)
tempstack = os.path.join(self.params['rundir'], "filter"+self.timestamp+".hed")
### first high pass filter particles
apDisplay.printMsg("pre-filtering particles")
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
boxsize = self.getBoxSize()
emancmd = ("proc2d "+emanstackfile+" "+tempstack
+" apix="+str(apix)+" ")
if self.params['highpasspart'] is not None and self.params['highpasspart'] > 0:
emancmd += "hp="+str(self.params['highpasspart'])+" "
if self.params['lowpasspart'] is not None and self.params['lowpasspart'] > 0:
emancmd += "lp="+str(self.params['lowpasspart'])+" "
if self.params['tiltbin'] > 1:
clipsize = boxsize*self.params['tiltbin']
emancmd += " shrink=%d clip=%d,%d "%(self.params['tiltbin'], clipsize, clipsize)
apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc2d "
emancmd += tempstack+" "
spiderstack = os.path.join(self.params['rundir'], "rctstack"+self.timestamp+".spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
time.sleep(1) # wait a sec, for things to finish
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
apFile.removeStack(tempstack, warn=False)
apFile.removeStack(emanstackfile, warn=False)
if not os.path.isfile(spiderstack):
apDisplay.printError("Failed to create a spider stack")
return spiderstack
def convertVolToSpider(self, mrcvolfile=None, modelid=None, apix=None, spivolfile=None):
"""
takes the mrc volume and creates a spider file ready for processing
"""
if modelid is not None:
initModelData = appiondata.ApInitialModelData.direct_query(modelid)
mrcvolfile = initModelData["path"]["path"] + "/" + initModelData["name"]
apix = initModelData["pixelsize"]
stackapix = apStack.getStackPixelSizeFromStackId(self.params["partstackid"]) * self.params["bin"]
stackboxsize = apStack.getStackBoxsize(self.params["partstackid"]) / self.params["bin"]
print apix
print mrcvolfile
if not os.path.isfile(mrcvolfile):
apDisplay.printError("volfile does not exist: " + mrcvolfile)
### first high pass filter particles
# apDisplay.printMsg("pre-filtering particles")
# apix = apStack.getStackPixelSizeFromStackId(self.params['partstackid'])
# emancmd = ("proc3d "+mrcvolfile+" "+emanstackfile
# +" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
# +" inplace")
# apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc3d "
emancmd += mrcvolfile + " "
if spivolfile is None:
spivolfile = os.path.join(self.params["rundir"], "threed-0a.spi")
apFile.removeFile(spivolfile, warn=True)
emancmd += spivolfile + " "
emancmd += "scale=" + str(apix / stackapix) + " "
emancmd += "clip=" + str(stackboxsize) + "," + str(stackboxsize) + "," + str(stackboxsize) + " "
emancmd += "spidersingle"
starttime = time.time()
apDisplay.printColor("Running spider volume conversion", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished conversion in " + apDisplay.timeString(time.time() - starttime), "cyan")
return spivolfile
def useStackForTemplate(self):
apDisplay.printMsg("Using stack to make templates")
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
self.params['sessionname'] = sessiondata['name']
self.params['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
stackdata = apStack.getOnlyStackData(self.params['stackid'])
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
self.params['templatelist'] = []
if self.params['avgstack'] is True:
### averaging all particles together
apDisplay.printMsg("Averaging all images in stack")
templatename = "stack%d-average.mrc"%(self.params['stackid'])
oldtemplatepath = os.path.join(stackdata['path']['path'], "average.mrc")
abstemplatepath = os.path.join(stackdata['path']['path'], templatename)
if os.path.isfile(oldtemplatepath):
shutil.copy(oldtemplatepath, abstemplatepath)
else:
### average all images using proc2d
emancmd = "proc2d %s %s average" % (stackfile, abstemplatepath)
apEMAN.executeEmanCmd(emancmd)
if os.path.isfile(abstemplatepath):
self.params['templatelist'] = [abstemplatepath]
elif self.params['imgnums'] is not None:
### check to see if stackimagenum is within the boundary of the stack
numpart = apFile.numImagesInStack(stackfile)
for i in self.params['imgnums'].split(","):
partnum = int(i)
if partnum > numpart or partnum < 0:
apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart))
apDisplay.printMsg("Extracting image %d from stack" % partnum)
templatename = "stack%d-particle%d.mrc"%(self.params['stackid'],partnum)
abstemplatepath= os.path.join(self.params['rundir'], templatename)
### run proc2d with params['stackimgnum']
emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum)
### create template
apDisplay.printMsg("creating "+templatename)
apEMAN.executeEmanCmd(emancmd)
if os.path.isfile(abstemplatepath):
self.params['templatelist'].append(abstemplatepath)
def start(self):
### load parameters
runparams = self.readRunParameters()
runparams['localstack'] = "start1.hdf"
self.params.update(runparams)
alignedClassStackHDF = "alignedClasses.hdf"
alignedPartStackHDF = "alignedParticles.hdf"
apFile.removeStack(alignedClassStackHDF, warn=False)
apFile.removeStack(alignedPartStackHDF, warn=False)
ISACParser = parseISAC()
ISACParser.setAlignParams(self.params)
ISACParser.trackParticlesInISAC()
### align classes
ISACParser.alignClassAverages(alignedClassStackHDF)
alignedClassStack = apEMAN2.stackHDFToIMAGIC(alignedClassStackHDF)
apStack.averageStack(alignedClassStack)
self.lastiter = ISACParser.numGenerations
### align particles to classes AND create aligned stacks
ISACParser.alignParticlesToClasses(self.params['localstack'],
alignedClassStackHDF,
alignedPartStackHDF)
alignedPartStack = apEMAN2.stackHDFToIMAGIC(alignedPartStackHDF)
self.numPart = sparx.EMUtil.get_image_count(alignedPartStackHDF)
partList = ISACParser.createPartList()
### calculate resolution for each reference
apix = apStack.getStackPixelSizeFromStackId(
self.params['stackid']) * self.params['bin']
self.calcResolution(partList, alignedPartStack, apix)
### insert into database
self.insertRunIntoDatabase(alignedPartStack, alignedClassStack,
self.params)
self.insertParticlesIntoDatabase(self.params['stackid'], partList)
def createFastFreeHandInputLineTemplate(self):
self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.apix = 6.02
one_particledata = apStack.getOneParticleFromStackId(self.params['stackid'], particlenumber=1, msg=True)
scopedata = one_particledata['particle']['image']['scope']
ht_kv = scopedata['high tension'] / 1000.0
cs_mm = scopedata['tem']['cs'] * 1000
angSearch = self.params['angSearch']
constant_inputlines = [
'%.3f,%.3f,%2f,%.f' %(self.apix,self.params['snr'],cs_mm,ht_kv)
,'1,0'
,'../%s' %(self.stackfile)
,'../%s' %(self.modelfile)
,'../%s' %(self.frealign_paramfile)
,'out.mrc'
,'-%d,%d,-%d,%d' %(angSearch,angSearch,angSearch,angSearch)
,'%.1f,%.2f,%.2f' %(self.params['minres'],self.params['maxres'],self.params['radius'])
,'%s' %(self.params['scoringtype'].upper())
]
return constant_inputlines
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['runname'] is None:
apDisplay.printError("new runname was not defined")
self.params['totalpart'] = apStack.getNumberStackParticlesFromId(
self.params['stackid'])
if 'last' not in self.params.keys() or self.params['last'] is None:
self.params['last'] = self.params['totalpart']
self.boxsize = apStack.getStackBoxsize(self.params['stackid'],
msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.refineboxsize = self.boxsize * self.params['bin']
### set cs value
self.params['cs'] = apInstrument.getCsValueFromSession(
self.getSessionData())
self.modelids = map((lambda x: int(x)),
self.params['modelid'].split(','))
self.checkPackageConflicts()
def checkConflicts(self):
if self.params['runname'] is None:
apDisplay.printError("enter a stack run name, e.g. combinestack1")
if self.params['description'] is None:
apDisplay.printError("enter a stack description")
if self.params['stacks'] and ',' in self.params['stacks']:
#remember stackids are a list of strings
stackids = self.params['stacks'].split(',')
self.params['stackids'] = stackids
else:
apDisplay.printError(
"enter a list of stack ids to combine, e.g. --stackids=11,14,7"
)
### check to make sure all pixel and box size are the same
self.newboxsize = None
self.newpixelsize = None
for stackidstr in self.params['stackids']:
if not re.match("^[0-9]+$", stackidstr):
apDisplay.printError("Stack id '%s' is not an integer" %
(stackidstr))
stackid = int(stackidstr)
boxsize = apStack.getStackBoxsize(stackid, msg=False)
pixelsize = apStack.getStackPixelSizeFromStackId(stackid,
msg=False)
apDisplay.printMsg("Stack id: %d\tBoxsize: %d\tPixelsize: %.3f" %
(stackid, boxsize, pixelsize))
if self.newboxsize is None:
self.newboxsize = boxsize
if self.newpixelsize is None:
self.newpixelsize = pixelsize
if boxsize != self.newboxsize:
apDisplay.printError(
"Trying to combine stacks with different box sizes")
if abs(pixelsize - self.newpixelsize) > 0.01:
apDisplay.printError(
"Trying to combine stacks with different pixel sizes")
def checkConflicts(self):
### first get all stack data
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### modify boxsize and pixelsize according to binning
self.boxsize = self.stack['boxsize']
self.clipsize = int(
math.floor(self.boxsize / float(self.params['bin'] * 2))) * 2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d" %
(self.params['clipsize'], self.clipsize))
self.clipsize = self.params['clipsize']
self.apix = self.stack['apix']
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
self.boxsize = clipsize
self.apix = self.apix * self.params['bin']
### basic error checking
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['ncls'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 1000000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles " +
str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(stackfile)))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > 5000:
apDisplay.printWarning(
"initial model calculation may not work with less than 5000 particles"
)
self.mpirun = self.checkMPI()
# if self.mpirun is None:
# apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
### SIMPLE defaults and error checking
if self.params['ring2'] is None:
self.params['ring2'] = (self.boxsize / 2) - 2
if self.params['ncls'] > 2000:
apDisplay.printError(
"number of classes should be less than 2000 for subsequent ORIGAMI run to work"
)
if self.params['ncls'] > self.params['numpart']:
self.params['ncls'] = self.params['numpart'] / self.params['minp']
if self.params['mask'] is None:
self.params['mask'] = (self.boxsize / 2) - 2
if self.params['mw'] is None:
apDisplay.printError(
"please specify the molecular weight (in kDa)")
def runEoTest(self, alignstack, eulerfile):
evenvolfile = os.path.join(self.params['rundir'],
"evenvolume%s.spi" % (self.timestamp))
oddvolfile = os.path.join(self.params['rundir'],
"oddvolume%s.spi" % (self.timestamp))
eveneulerfile = os.path.join(self.params['rundir'],
"eveneulers%s.spi" % (self.timestamp))
oddeulerfile = os.path.join(self.params['rundir'],
"oddeulers%s.spi" % (self.timestamp))
evenpartlist = os.path.join(self.params['rundir'],
"evenparts%s.lst" % (self.timestamp))
oddpartlist = os.path.join(self.params['rundir'],
"oddparts%s.lst" % (self.timestamp))
### Create New Doc Files
of = open(oddeulerfile, "w")
ef = open(eveneulerfile, "w")
op = open(oddpartlist, "w")
ep = open(evenpartlist, "w")
inf = open(eulerfile, "r")
evenpart = 0
oddpart = 0
for line in inf:
spidict = operations.spiderInLine(line)
if spidict:
partnum = spidict['row']
if partnum % 2 == 0:
ep.write("%d\n" % (partnum - 1))
evenpart += 1
outline = operations.spiderOutLine(evenpart,
spidict['floatlist'])
ef.write(outline)
elif partnum % 2 == 1:
op.write("%d\n" % (partnum - 1))
oddpart += 1
outline = operations.spiderOutLine(oddpart,
spidict['floatlist'])
of.write(outline)
inf.close()
of.close()
ef.close()
op.close()
ep.close()
### Create stacks
evenstack = os.path.join(self.params['rundir'],
"evenstack%s.spi" % (self.timestamp))
emancmd = "proc2d %s %s list=%s spiderswap" % (alignstack, evenstack,
evenpartlist)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
oddstack = os.path.join(self.params['rundir'],
"oddstack%s.spi" % (self.timestamp))
emancmd = "proc2d %s %s list=%s spiderswap" % (alignstack, oddstack,
oddpartlist)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
### Create Volumes
backproject.backproject3F(evenstack, eveneulerfile, evenvolfile,
evenpart)
backproject.backproject3F(oddstack, oddeulerfile, oddvolfile, oddpart)
if not os.path.isfile(evenvolfile) or not os.path.isfile(oddvolfile):
apDisplay.printError("Even-Odd volume creation failed")
### Calculate FSC
apix = apStack.getStackPixelSizeFromStackId(
self.params['tiltstackid']) * self.params['tiltbin']
emancmd = "proc3d %s %s" % (evenvolfile, evenvolfile + ".mrc")
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
emancmd = "proc3d %s %s" % (oddvolfile, oddvolfile + ".mrc")
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
fscfile = os.path.join(self.params['rundir'],
"fscdata%s.fsc" % (self.timestamp))
emancmd = "proc3d %s %s fsc=%s" % (evenvolfile + ".mrc",
oddvolfile + ".mrc", fscfile)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
if not os.path.isfile(fscfile):
apDisplay.printError("Even-Odd fsc calculation failed")
boxsize = self.getBoxSize()
self.fscresolution = apRecon.getResolutionFromFSCFile(fscfile,
boxsize,
apix,
msg=True)
apDisplay.printColor(
("Final FSC resolution: %.5f" % (self.fscresolution)), "cyan")
for fname in (evenvolfile, oddvolfile, evenstack, oddstack,
eveneulerfile, oddeulerfile, evenpartlist, oddpartlist):
apFile.removeFile(fname)
