def checkConflicts(self):
if self.params["stackid"] is None:
apDisplay.printError("stack id was not defined")
if self.params["modelid"] is None:
apDisplay.printError("model id was not defined")
if self.params["runname"] is None:
apDisplay.printError("missing a reconstruction name")
if self.params["last"] is None:
self.params["last"] = apStack.getNumberStackParticlesFromId(self.params["stackid"])
self.boxsize = apStack.getStackBoxsize(self.params["stackid"], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
if self.params["reconstackid"] is not None:
reconboxsize = apStack.getStackBoxsize(self.params["reconstackid"], msg=False)
reconapix = apStack.getStackPixelSizeFromStackId(self.params["reconstackid"])
refinenumpart = apStack.getNumberStackParticlesFromId(self.params["stackid"])
reconnumpart = apStack.getNumberStackParticlesFromId(self.params["reconstackid"])
if reconboxsize != self.boxsize:
apDisplay.printError("Boxsize do not match for stacks")
if reconapix != self.apix:
apDisplay.printError("Pixelsize do not match for stacks")
if refinenumpart != reconnumpart:
apDisplay.printError("Number of particles do not match for stacks")
maxmask = math.floor(self.apix * (self.boxsize - 10) / 2.0)
if self.params["mask"] is None:
apDisplay.printWarning("mask was not defined, setting to boxsize: %d" % (maxmask))
self.params["mask"] = maxmask
if self.params["mask"] > maxmask:
apDisplay.printWarning("mask was too big, setting to boxsize: %d" % (maxmask))
self.params["mask"] = maxmask
if self.params["noctf"] is True:
apDisplay.printWarning("Using no CTF method")
self.params["wgh"] = -1.0
if self.params["nodes"] > 1 and self.params["cluster"] is False:
apDisplay.printError("cluster mode must be enabled to run on more than 1 node")
if self.params["ppn"] is None:
if self.params["cluster"] is True:
apDisplay.printError("you must define ppn for cluster mode")
self.params["ppn"] = apParam.getNumProcessors()
apDisplay.printMsg("Setting number of processors to %d" % (self.params["ppn"]))
if self.params["rpn"] is None:
self.params["rpn"] = self.params["ppn"]
self.params["nproc"] = self.params["nodes"] * self.params["rpn"]
### get the symmetry data
if self.params["sym"] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params["sym"])
self.params["symm_id"] = self.symmdata.dbid
self.params["symm_name"] = self.symmdata["eman_name"]
apDisplay.printMsg("Selected symmetry %s with id %s" % (self.symmdata["eman_name"], self.symmdata.dbid))
### set cs value
self.params["cs"] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['runname'] is None:
apDisplay.printError("missing a reconstruction name")
if self.params['last'] is None:
self.params['last'] = apStack.getNumberStackParticlesFromId(self.params['stackid'])
self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
if self.params['reconstackid'] is not None:
reconboxsize = apStack.getStackBoxsize(self.params['reconstackid'], msg=False)
reconapix = apStack.getStackPixelSizeFromStackId(self.params['reconstackid'])
refinenumpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
reconnumpart = apStack.getNumberStackParticlesFromId(self.params['reconstackid'])
if reconboxsize != self.boxsize:
apDisplay.printError("Boxsize do not match for stacks")
if reconapix != self.apix:
apDisplay.printError("Pixelsize do not match for stacks")
if refinenumpart != reconnumpart:
apDisplay.printError("Number of particles do not match for stacks")
maxmask = math.floor(self.apix*(self.boxsize-10)/2.0)
if self.params['mask'] is None:
apDisplay.printWarning("mask was not defined, setting to boxsize: %d"%(maxmask))
self.params['mask'] = maxmask
if self.params['mask'] > maxmask:
apDisplay.printWarning("mask was too big, setting to boxsize: %d"%(maxmask))
self.params['mask'] = maxmask
if self.params['noctf'] is True:
apDisplay.printWarning("Using no CTF method")
self.params['wgh'] = -1.0
if self.params['nodes'] > 1 and self.params['cluster'] is False:
apDisplay.printError("cluster mode must be enabled to run on more than 1 node")
if self.params['ppn'] is None:
if self.params['cluster'] is True:
apDisplay.printError("you must define ppn for cluster mode")
self.params['ppn'] = apParam.getNumProcessors()
apDisplay.printMsg("Setting number of processors to %d"%(self.params['ppn']))
if self.params['rpn'] is None:
self.params['rpn'] = self.params['ppn']
self.params['nproc'] = self.params['nodes']*self.params['rpn']
### get the symmetry data
if self.params['sym'] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params['sym'])
self.params['symm_id'] = self.symmdata.dbid
self.params['symm_name'] = self.symmdata['eman_name']
apDisplay.printMsg("Selected symmetry %s with id %s"%(self.symmdata['eman_name'], self.symmdata.dbid))
### set cs value
self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("Please provide a user id, e.g. --stackid=15")
if not 'radonshift' in dir(radermacher):
apDisplay.printError("Radermacher python module is too old, please update")
boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
pixelsize = apStack.getStackPixelSizeFromStackId(self.params['stackid'], msg=False)
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'], msg=False)
apDisplay.printMsg("\n\n")
### get info about the stack
apDisplay.printMsg("Information about stack id %d"%(self.params['stackid']))
apDisplay.printMsg("\tboxsize %d pixels"%(boxsize))
apDisplay.printMsg("\tpixelsize %.3f Angstroms"%(pixelsize))
apDisplay.printMsg("\tsize %d particles"%(numpart))
if self.params['maskrad'] is None:
self.params['pixelmaskrad'] = boxsize/2
self.params['maskrad'] = self.params['pixelmaskrad']*pixelsize
else:
self.params['pixelmaskrad'] = self.params['maskrad']/pixelsize
if self.params['pixelmaskrad'] > boxsize/2:
apDisplay.printError("Mask radius larger than boxsize")
if self.params['numpart'] is None:
self.params['numpart'] = numpart
if self.params['numpart'] > numpart:
apDisplay.printError("Requested more particles than available in stack")
if self.params['numrefs'] is None:
### num refs should go approx. as the sqrt of the number of particles
self.params['numrefs'] = int(math.ceil(math.sqrt(self.params['numpart'])))+1
def start(self):
### new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
### make sure that old stack is numbered
alignedstack = os.path.join(self.params['rundir'], 'alignstack.hed')
apStack.checkForPreviousStack(alignedstack)
### run centering algorithm
self.runMaxlike()
### create aligned stacks
partlist = self.readPartDocFile()
stackfile = self.createAlignedStacks(partlist)
if not os.path.isfile(alignedstack):
apDisplay.printError("No stack was created")
### get number of particles
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
self.writeFakeKeepFile(numpart)
self.params['description'] += (
" ... %d maxlike centered substack id %d"
% (numpart, self.params['stackid']))
apStack.commitSubStack(self.params, newname='alignstack.hed', centered=True)
apStack.averageStack(stack=alignedstack)
def start(self):
### new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
### make sure that old stack is numbered
alignedstack = os.path.join(self.params['rundir'], 'alignstack.hed')
apStack.checkForPreviousStack(alignedstack)
### run centering algorithm
self.runMaxlike()
### create aligned stacks
partlist = self.readPartDocFile()
stackfile = self.createAlignedStacks(partlist)
if not os.path.isfile(alignedstack):
apDisplay.printError("No stack was created")
### get number of particles
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
self.writeFakeKeepFile(numpart)
self.params['description'] += (
" ... %d maxlike centered substack id %d" %
(numpart, self.params['stackid']))
apStack.commitSubStack(self.params,
newname='alignstack.hed',
centered=True)
apStack.averageStack(stack=alignedstack)
def getGoodParticles(self):
'''
Get particle numbers in eman format in fullstack that
has tilt pair in knownstack
'''
includeParticle = []
tiltParticlesData = []
no_match_count = 0
apDisplay.printMsg("Finding particles from tilt pairs ")
### get stack data
knownstackid = self.params['knownstackid']
fullstackid = self.params['fullstackid']
numpartl = apStack.getNumberStackParticlesFromId(self.params['knownstackid'])
for partnum in range(1,numpartl+1):
stpartdata = apStack.getStackParticle(knownstackid, partnum, True)
# stpartdata and otherpartdata are related by tilt transform
imgnum, transformdata, otherpartdata = apTiltPair.getTiltTransformFromParticle(stpartdata['particle'])
apDisplay.printMsg('Mapping particle %d to %d' % (stpartdata['particle'].dbid,otherpartdata.dbid))
fullstackpartdata = apStack.getStackParticleFromData(fullstackid, otherpartdata, True)
if fullstackpartdata:
# eman particle number starts from 0, appion database number starts from 1
emantiltstackpartnum = fullstackpartdata['particleNumber']-1
includeParticle.append(emantiltstackpartnum)
tiltParticlesData.append(fullstackpartdata)
else:
no_match_count += 1
apDisplay.printWarning('There are %d particles without a match in the requested full stack' % no_match_count)
return includeParticle, tiltParticlesData
def partnum2defid(stackid):
"""
This function must be used because sinedon would take up too much memory?
"""
t0 = time.time()
stackpartnum = apStack.getNumberStackParticlesFromId(stackid)
apDisplay.printMsg("Mapping %d stack particle IDs"%(stackpartnum))
import sinedon
import MySQLdb
dbconf = sinedon.getConfig('appiondata')
db = MySQLdb.connect(**dbconf)
cursor = db.cursor()
cursor.execute('SELECT DEF_id, particleNumber FROM ApStackParticleData WHERE `REF|ApStackData|stack` = %s' % (stackid,))
partdict = {}
maxnum = 0
while True:
row = cursor.fetchone()
if row is None:
break
defid = int(row[0]) #row['DEF_id']
num = int(row[1]) #row['particleNumber']
if num > maxnum:
maxnum = num
partdict[num] = defid
if stackpartnum != maxnum:
apDisplay.printError("Expected to get %d particles, but received %d particles"%(stackpartnum, maxnum))
apDisplay.printMsg("Mapped %d stack particle IDs in %s"%(stackpartnum, apDisplay.timeString(time.time()-t0)))
return partdict
def writeMultipleFreeHandTestShells(self, constant_inputlines, nproc):
scripts = []
files = []
last_particle = apStack.getNumberStackParticlesFromId(self.params["stackid"], msg=True)
last_particle = 82
# This is integer division and will return integer
stepsize = int(math.ceil(float(last_particle) / nproc))
for proc in range(nproc):
proc_start_particle = stepsize * proc + 1
proc_end_particle = min(stepsize * (proc + 1), last_particle)
proc_inputlines = list(constant_inputlines)
proc_inputlines.insert(8, "%d, %d" % (proc_start_particle, proc_end_particle))
proc_inputlines.extend(self.createTiltedCtfLines(proc_start_particle, proc_end_particle))
procpath = os.path.join(self.params["rundir"], "proc%03d" % (proc))
apParam.createDirectory(procpath, warning=False)
lines_before_input = [
"#!/bin/csh",
"# Proc %03d, Particles %d - %d" % (proc, proc_start_particle, proc_end_particle),
"rm -rf %s" % procpath,
"mkdir %s" % procpath,
"cd %s" % procpath,
"",
"",
"### START FREALIGN ###",
"/home/acheng/bin/fastfreehand_v1_01.exe << EOF > freehand.proc%03d.out" % proc,
]
lines_after_input = ["EOF", "", "### END FREEHAND", 'echo "END FREEHAND"']
alllines = lines_before_input + proc_inputlines + lines_after_input
procfile = "freehand.proc%03d.csh" % (proc)
f = open(procfile, "w")
f.writelines(map((lambda x: x + "\n"), alllines))
f.close()
os.chmod(procfile, 0755)
scripts.append(procfile)
return scripts
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if self.params['description'] is None:
apDisplay.printError("substack description was not defined")
if self.params['runname'] is None:
apDisplay.printError("new stack name was not defined")
### have first but not last
if self.params['first'] is not None and self.params['last'] is None:
if self.params['random'] is not None:
apDisplay.printError(
"Random function can't combine with range of the stack")
self.params['last'] = numpart
elif self.params['first'] is None and self.params['last'] is not None:
if self.params['random'] is not None:
apDisplay.printError(
"Random function can't combine with range of the stack")
# This is spider-style (1-....)
self.params['first'] = 1
if self.params['last'] is None:
self.params['last'] = numpart
if self.params['random'] > numpart or self.params['last'] > numpart:
apDisplay.printError(
"Requesting too many particles from stack, max %d" % (numpart))
if self.params['first'] is None and self.params['split'] == 1:
if self.params['keepfile'] is None and self.params[
'exclude'] is None and self.params[
'include'] is None and self.params['random'] is None:
apDisplay.printError(
"Please define either keepfile, exclude or include")
elif self.params['keepfile']:
self.params['keepfile'] = os.path.abspath(
self.params['keepfile'])
if not os.path.isfile(self.params['keepfile']):
apDisplay.printError("Could not find keep file: " +
self.params['keepfile'])
if self.params['keepfile'] and (self.params['exclude'] is not None
or self.params['include'] is not None):
apDisplay.printError(
"Please define only either keepfile, exclude or include")
if self.params['exclude'] and (self.params['keepfile'] is not None
or self.params['include'] is not None):
apDisplay.printError(
"Please define only either keepfile, exclude or include")
if self.params['include'] and (self.params['exclude'] is not None
or self.params['keepfile'] is not None):
apDisplay.printError(
"Please define only either keepfile, exclude or include")
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("Please provide a stack id, e.g. --stackid=15")
if self.params['runname'] is None:
apDisplay.printError("Please provide a run name id, e.g. --runname=refine1")
if self.params['description'] is None:
apDisplay.printError("Please provide a description, e.g. --description='something'")
### get number of particles in stack
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if self.params['numpart'] is None:
self.params['numpart'] = numpart
elif self.params['numpart'] > numpart:
apDisplay.printError("Requested number of particles (%d) is greater than the number of particles in the stack (%d)"%(self.params['numpart'], numpart))
def boxMask(self,infile,outfile,spirots=None):
from appionlib.apSpider import operations
# boxmask the particles
apDisplay.printMsg("masking the particles with a rectangular box")
nump = apStack.getNumberStackParticlesFromId(self.params['stackid'])
box = self.stackdata['boxsize']
apix = self.stackdata['pixelsize']*1e10
if self.params['mask'] is None:
mask = box/2-2
else:
mask = int(self.params['mask']/apix)
imask = int(self.params['imask']/apix)
length = int(self.params['length']/apix)
falloff = self.params['falloff']/apix
mask -= falloff/2
length = (length/2)-(falloff/2)
# create blank image for mask
maskfile = "boxmask.spi"
operations.createBoxMask(maskfile,box,mask,length,falloff,imask)
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
mySpi.toSpiderQuiet("CP",
spyder.fileFilter(maskfile),"_4")
mySpi.toSpider("do x10=1,%i"%nump)
if self.params['vertical'] is not True:
mySpi.toSpider("UD IC x10,x30",
spyder.fileFilter(spirots),
"x30 = -1*x30",
"RT B",
"_4",
"_9",
"(x30)",
"(0)",
"MU",
spyder.fileFilter(infile)+"@{******x10}",
"_9")
else:
mySpi.toSpider("MU",
spyder.fileFilter(infile)+"@{******x10}",
"_4")
mySpi.toSpider(spyder.fileFilter(outfile)+"@{******x10}",
"*",
"enddo")
if self.params['vertical'] is not True:
mySpi.toSpider("UD ICE",spyder.fileFilter(spirots))
mySpi.close()
def writeMultipleFreeHandTestShells(self, constant_inputlines, nproc):
scripts = []
files = []
last_particle = apStack.getNumberStackParticlesFromId(
self.params['stackid'], msg=True)
last_particle = 82
# This is integer division and will return integer
stepsize = int(math.ceil(float(last_particle) / nproc))
for proc in range(nproc):
proc_start_particle = stepsize * proc + 1
proc_end_particle = min(stepsize * (proc + 1), last_particle)
proc_inputlines = list(constant_inputlines)
proc_inputlines.insert(
8, '%d, %d' % (proc_start_particle, proc_end_particle))
proc_inputlines.extend(
self.createTiltedCtfLines(proc_start_particle,
proc_end_particle))
procpath = os.path.join(self.params['rundir'], 'proc%03d' % (proc))
apParam.createDirectory(procpath, warning=False)
lines_before_input = [
'#!/bin/csh',
'# Proc %03d, Particles %d - %d' %
(proc, proc_start_particle, proc_end_particle),
'rm -rf %s' % procpath,
'mkdir %s' % procpath,
'cd %s' % procpath,
'',
'',
'### START FREALIGN ###',
'/home/acheng/bin/fastfreehand_v1_01.exe << EOF > freehand.proc%03d.out'
% proc,
]
lines_after_input = [
'EOF',
'',
'### END FREEHAND',
'echo "END FREEHAND"',
]
alllines = lines_before_input + proc_inputlines + lines_after_input
procfile = 'freehand.proc%03d.csh' % (proc)
f = open(procfile, 'w')
f.writelines(map((lambda x: x + '\n'), alllines))
f.close()
os.chmod(procfile, 0755)
scripts.append(procfile)
return scripts
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['runname'] is None:
apDisplay.printError("new runname was not defined")
self.params['totalpart'] = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if 'last' not in self.params.keys() or self.params['last'] is None:
self.params['last'] = self.params['totalpart']
self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.refineboxsize = self.boxsize * self.params['bin']
### set cs value
self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
self.modelids = map((lambda x: int(x)),self.params['modelid'].split(','))
self.checkPackageConflicts()
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError(
"Please provide a stack id, e.g. --stackid=15")
if self.params['runname'] is None:
apDisplay.printError(
"Please provide a run name id, e.g. --runname=refine1")
if self.params['description'] is None:
apDisplay.printError(
"Please provide a description, e.g. --description='something'")
### get number of particles in stack
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if self.params['numpart'] is None:
self.params['numpart'] = numpart
elif self.params['numpart'] > numpart:
apDisplay.printError(
"Requested number of particles (%d) is greater than the number of particles in the stack (%d)"
% (self.params['numpart'], numpart))
def boxMask(self, infile, outfile, spirots=None):
from appionlib.apSpider import operations
# boxmask the particles
apDisplay.printMsg("masking the particles with a rectangular box")
nump = apStack.getNumberStackParticlesFromId(self.params['stackid'])
box = self.stackdata['boxsize']
apix = self.stackdata['pixelsize'] * 1e10
if self.params['mask'] is None:
mask = box / 2 - 2
else:
mask = int(self.params['mask'] / apix)
imask = int(self.params['imask'] / apix)
length = int(self.params['length'] / apix)
falloff = self.params['falloff'] / apix
mask -= falloff / 2
length = (length / 2) - (falloff / 2)
# create blank image for mask
maskfile = "boxmask.spi"
operations.createBoxMask(maskfile, box, mask, length, falloff, imask)
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
mySpi.toSpiderQuiet("CP", spyder.fileFilter(maskfile), "_4")
mySpi.toSpider("do x10=1,%i" % nump)
if self.params['vertical'] is not True:
mySpi.toSpider("UD IC x10,x30", spyder.fileFilter(spirots),
"x30 = -1*x30", "RT B", "_4", "_9", "(x30)", "(0)",
"MU",
spyder.fileFilter(infile) + "@{******x10}", "_9")
else:
mySpi.toSpider("MU",
spyder.fileFilter(infile) + "@{******x10}", "_4")
mySpi.toSpider(
spyder.fileFilter(outfile) + "@{******x10}", "*", "enddo")
if self.params['vertical'] is not True:
mySpi.toSpider("UD ICE", spyder.fileFilter(spirots))
mySpi.close()
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if self.params['description'] is None:
apDisplay.printError("substack description was not defined")
if self.params['runname'] is None:
apDisplay.printError("new stack name was not defined")
### have first but not last
if self.params['first'] is not None and self.params['last'] is None:
if self.params['random'] is not None:
apDisplay.printError("Random function can't combine with range of the stack")
self.params['last'] = numpart
elif self.params['first'] is None and self.params['last'] is not None:
if self.params['random'] is not None:
apDisplay.printError("Random function can't combine with range of the stack")
# This is spider-style (1-....)
self.params['first'] = 1
if self.params['last'] is None:
self.params['last'] = numpart
if self.params['random'] > numpart or self.params['last'] > numpart:
apDisplay.printError("Requesting too many particles from stack, max %d"%(numpart))
if self.params['first'] is None and self.params['split'] == 1:
if self.params['keepfile'] is None and self.params['exclude'] is None and self.params['include'] is None and self.params['random'] is None:
apDisplay.printError("Please define either keepfile, exclude or include")
elif self.params['keepfile']:
self.params['keepfile'] = os.path.abspath(self.params['keepfile'])
if not os.path.isfile(self.params['keepfile']):
apDisplay.printError("Could not find keep file: "+self.params['keepfile'])
if self.params['keepfile'] and (self.params['exclude'] is not None or self.params['include'] is not None):
apDisplay.printError("Please define only either keepfile, exclude or include")
if self.params['exclude'] and (self.params['keepfile'] is not None or self.params['include'] is not None):
apDisplay.printError("Please define only either keepfile, exclude or include")
if self.params['include'] and (self.params['exclude'] is not None or self.params['keepfile'] is not None):
apDisplay.printError("Please define only either keepfile, exclude or include")
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['runname'] is None:
apDisplay.printError("new runname was not defined")
self.params['totalpart'] = apStack.getNumberStackParticlesFromId(
self.params['stackid'])
if 'last' not in self.params.keys() or self.params['last'] is None:
self.params['last'] = self.params['totalpart']
self.boxsize = apStack.getStackBoxsize(self.params['stackid'],
msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.refineboxsize = self.boxsize * self.params['bin']
### set cs value
self.params['cs'] = apInstrument.getCsValueFromSession(
self.getSessionData())
self.modelids = map((lambda x: int(x)),
self.params['modelid'].split(','))
self.checkPackageConflicts()
def convertToRefineStack(self):
'''
The stack is remaked without ctf correction and without invertion (ccd)
'''
newstackroot = os.path.join(self.params['rundir'],os.path.basename(self.stack['file'])[:-4])
if self.params['paramonly'] is True:
print 'newstackroot',newstackroot
self.setFrealignStack(newstackroot)
return
# copy existing refined stack if possible
existing_refine_stack = apStack.getExistingRefineStack(self.stack['data'],'frealign',False,self.params['last'],self.params['bin'],lowpass=self.params['lowpass'],highpass=self.params['highpass'])
if existing_refine_stack:
if existing_refine_stack != os.path.join(self.params['rundir'],newstackroot+'.mrc'):
shutil.copyfile(existing_refine_stack,newstackroot+'.mrc')
new_partorderfile = os.path.join(self.params['rundir'],'stackpartorder.list')
existing_partorderfile = os.path.join(os.path.dirname(existing_refine_stack),'stackpartorder.list')
# particle order list also need to be copied
if not os.path.isfile(new_partorderfile) and os.path.isfile(existing_partorderfile):
shutil.copyfile(existing_partorderfile,new_partorderfile)
self.setFrealignStack(newstackroot)
return
if self.no_ctf_correction:
self.ImagicStackToFrealignMrcStack(self.stack['file'])
self.setFrealignStack(newstackroot)
return
# stack need to be remade without ctf correction
apDisplay.printWarning('Frealign needs a stack without ctf correction. A new stack is being made....')
stackdata = self.stack['data']
madestackdata = self.getOriginalStackMadeByMakeStack()
stackid = stackdata.dbid
stackruns = apStack.getStackRunsFromStack(madestackdata)
# To Do: need to handle combined stack
stackrun = stackruns[0]
stackpathname = os.path.basename(stackdata['path']['path'])
totalpart = apStack.getNumberStackParticlesFromId(stackid)
if not self.params['last']:
numpart = totalpart
else:
numpart = min(self.params['last'],totalpart)
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir,'temp.hed')
# use first particle image to get presetname
oneparticle = apStack.getOneParticleFromStackId(stackid, particlenumber=1)
preset =oneparticle['particle']['image']['preset']
if preset:
presetname = preset['name']
else:
presetname = 'manual'
# use first stack run to get parameters
paramdata = stackrun['stackParams']
# binning is combination of the original binning of the stack and the preparation binnning
bin = paramdata['bin']*self.params['bin']
unbinnedboxsize = self.stack['boxsize']*paramdata['bin']
lowpasstext = self.setArgText('lowpass',(self.params['lowpass'],paramdata['lowpass']),False)
highpasstext = self.setArgText('highpass',(self.params['highpass'],paramdata['highpass']),True)
partlimittext = self.setArgText('partlimit',(numpart,),False)
xmipp_normtext = self.setArgText('xmipp-normalize',(paramdata['xmipp-norm'],),True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
# The assumption is that the image is from ice grid and digital camera (black particles on white background
if 'reverse' in self.stackrunlogparams.keys():
reversetext = '--reverse'
else:
reversetext = ''
if 'defocpair' in self.stackrunlogparams.keys():
defoctext = '--defocpair'
else:
defoctext = ''
cmd = '''
makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s --no-invert --normalized %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2
''' % (os.path.basename(newstackimagicfile),stackid,lowpasstext,highpasstext,partlimittext,reversetext,defoctext,xmipp_normtext,unbinnedboxsize,bin,stackpathname,stackid,projectid,presetname,newstackrunname,newstackrundir,sessionname,sessionid)
logfilepath = os.path.join(newstackrundir,'frealignstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd,logfilepath)
if returncode > 0:
apDisplay.printError('Error in Frealign specific stack making')
self.ImagicStackToFrealignMrcStack(newstackimagicfile)
os.rename(newstackimagicfile[:-4]+'.mrc',newstackroot+'.mrc')
self.setFrealignStack(newstackroot)
# use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
#clean up
rmfiles = glob.glob("*.box")
for rmfile in rmfiles:
apFile.removeFile(rmfile)
def convertToRefineStack(self):
'''
The stack is remaked without ctf correction and without invertion (ccd)
'''
newstackroot = os.path.join(self.params['rundir'],
os.path.basename(self.stack['file'])[:-4])
if self.params['paramonly'] is True:
print 'newstackroot', newstackroot
self.setFrealignStack(newstackroot)
return
# copy existing refined stack if possible
existing_refine_stack = apStack.getExistingRefineStack(
self.stack['data'],
'frealign',
False,
self.params['last'],
self.params['bin'],
lowpass=self.params['lowpass'],
highpass=self.params['highpass'])
try:
if existing_refine_stack:
if existing_refine_stack != os.path.join(
self.params['rundir'], newstackroot + '.mrc'):
shutil.copyfile(existing_refine_stack,
newstackroot + '.mrc')
new_partorderfile = os.path.join(self.params['rundir'],
'stackpartorder.list')
existing_partorderfile = os.path.join(
os.path.dirname(existing_refine_stack),
'stackpartorder.list')
# particle order list also need to be copied
if not os.path.isfile(new_partorderfile) and os.path.isfile(
existing_partorderfile):
shutil.copyfile(existing_partorderfile, new_partorderfile)
self.setFrealignStack(newstackroot)
return
except:
# make one from scratch if there is error
apDisplay.printMsg(
'Error copying existing refine stack, will regenerate')
if self.no_ctf_correction:
self.ImagicStackToFrealignMrcStack(self.stack['file'])
self.setFrealignStack(newstackroot)
return
# stack need to be remade without ctf correction
apDisplay.printWarning(
'Frealign needs a stack without ctf correction. A new stack is being made....'
)
stackdata = self.stack['data']
madestackdata = self.getOriginalStackMadeByMakeStack()
stackid = stackdata.dbid
stackruns = apStack.getStackRunsFromStack(madestackdata)
# To Do: need to handle combined stack
stackrun = stackruns[0]
stackpathname = os.path.basename(stackdata['path']['path'])
totalpart = apStack.getNumberStackParticlesFromId(stackid)
if not self.params['last']:
numpart = totalpart
else:
numpart = min(self.params['last'], totalpart)
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir, 'temp.hed')
# use first particle image to get presetname
oneparticle = apStack.getOneParticleFromStackId(stackid,
particlenumber=1)
preset = oneparticle['particle']['image']['preset']
if preset:
presetname = preset['name']
else:
presetname = 'manual'
# use first stack run to get parameters
paramdata = stackrun['stackParams']
# binning is combination of the original binning of the stack and the preparation binnning
bin = paramdata['bin'] * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * paramdata['bin']
lowpasstext = self.setArgText(
'lowpass', (self.params['lowpass'], paramdata['lowpass']), False)
highpasstext = self.setArgText(
'highpass', (self.params['highpass'], paramdata['highpass']), True)
partlimittext = self.setArgText('partlimit', (numpart, ), False)
#normalization
normtext = ''
if 'xmipp-norm' in paramdata.keys():
# myami-3.1 or before
normtext = self.setArgText('xmipp-normalize',
(paramdata['xmipp-norm'], ), True)
normtext = '--normalized ' + normtext
elif 'normalizemethod' in paramdata.keys():
# myami-3.2 or newer
normtext = '--normalize-method=%s' % (
paramdata['normalizemethod'], )
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
# The assumption is that the image is from ice grid and digital camera (black particles on white background
if 'reverse' in self.stackrunlogparams.keys():
reversetext = '--reverse'
else:
reversetext = ''
if 'defocpair' in self.stackrunlogparams.keys():
defoctext = '--defocpair'
else:
defoctext = ''
cmd = '''
makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s --no-invert %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2
''' % (os.path.basename(newstackimagicfile), stackid, lowpasstext,
highpasstext, partlimittext, reversetext, defoctext, normtext,
unbinnedboxsize, bin, stackpathname, stackid, projectid, presetname,
newstackrunname, newstackrundir, sessionname, sessionid)
logfilepath = os.path.join(newstackrundir, 'frealignstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd, logfilepath)
if returncode > 0:
apDisplay.printError('Error in Frealign specific stack making')
self.ImagicStackToFrealignMrcStack(newstackimagicfile)
os.rename(newstackimagicfile[:-4] + '.mrc', newstackroot + '.mrc')
self.setFrealignStack(newstackroot)
# use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
#clean up
rmfiles = glob.glob("*.box")
for rmfile in rmfiles:
apFile.removeFile(rmfile)
def start(self):
#old stack size
stacksize = apStack.getNumberStackParticlesFromId(self.params['stackid'])
# if exclude or include lists are not defined...
if self.params['exclude'] is None and self.params['include'] is None:
# if first and last are specified, create a file
if self.params['first'] is not None and self.params['last'] is not None:
stp = str(self.params['first'])
enp = str(self.params['last'])
fname = 'sub'+str(self.params['stackid'])+'_'+stp+'-'+enp+'.lst'
self.params['keepfile'] = os.path.join(self.params['rundir'],fname)
apDisplay.printMsg("Creating keep list: "+self.params['keepfile'])
f=open(self.params['keepfile'],'w')
for i in range(self.params['first'],self.params['last']+1):
f.write('%d\n' % (int(i)-1))
f.close()
# generate the random list by giving number and create the file
elif self.params['random'] is not None:
#numOfRandomParticles = str(self.params['random'])
#fname = 'random'+str(self.params['stackid'])+'_'+numOfRandomParticles+'.lst'
fname = "random%d_%d.lst"%(self.params['stackid'], self.params['random'])
self.params['keepfile'] = os.path.join(self.params['rundir'],fname)
apDisplay.printMsg("Creating keep list: "+self.params['keepfile'])
# create a file
f=open(self.params['keepfile'],'w')
# generate a random sequence by giving size
randomList = random.sample(xrange(self.params['last']), self.params['random'])
randomList.sort()
for partnum in randomList:
f.write('%d\n' % partnum)
f.close()
# if splitting, create files containing the split values
elif self.params['split'] > 1:
for i in range(self.params['split']):
fname = 'sub'+str(self.params['stackid'])+'.'+str(i+1)+'.lst'
self.params['keepfile'] = os.path.join(self.params['rundir'],fname)
apDisplay.printMsg("Creating keep list: "+self.params['keepfile'])
f = open(self.params['keepfile'],'w')
for p in range(stacksize):
if (p % self.params['split'])-i==0:
f.write('%i\n' % p)
f.close()
# if exclude-from option is specified, convert particles to exclude
elif self.params['excludefile'] is True:
oldkf = open(self.params['keepfile'])
partlist = []
for line in oldkf:
particle=line.strip()
try:
particle = int(particle)
except:
continue
partlist.append(particle)
oldkf.close()
# create new list excluding the particles
apDisplay.printMsg("Converting keep file to exclude file")
newkeepfile = "tmpnewkeepfile.txt"
newkf = open(newkeepfile,'w')
for p in range(stacksize):
if p not in partlist:
newkf.write("%i\n"%p)
newkf.close()
self.params['keepfile'] = os.path.abspath(newkeepfile)
# otherwise, just copy the file
elif not os.path.isfile(os.path.basename(self.params['keepfile'])):
shutil.copy(self.params['keepfile'], os.path.basename(self.params['keepfile']))
# if either exclude or include lists is defined
elif self.params['exclude'] or self.params['include']:
### list of particles to be excluded
excludelist = []
if self.params['exclude'] is not None:
excludestrlist = self.params['exclude'].split(",")
for excld in excludestrlist:
excludelist.append(int(excld.strip()))
apDisplay.printMsg("Exclude list: "+str(excludelist))
### list of particles to be included
includelist = []
if self.params['include'] is not None:
includestrlist = self.params['include'].split(",")
for incld in includestrlist:
includelist.append(int(incld.strip()))
apDisplay.printMsg("Include list: "+str(includelist))
#new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
newname = stackdata['name']
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
#if include or exclude list is given...
if self.params['include'] is not None or self.params['exclude'] is not None:
includeParticle = []
excludeParticle = 0
for partnum in range(stacksize):
if includelist and partnum in includelist:
includeParticle.append(partnum)
elif excludelist and not partnum in excludelist:
includeParticle.append(partnum)
else:
excludeParticle += 1
includeParticle.sort()
### write kept particles to file
self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list")
apDisplay.printMsg("writing to keepfile "+self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
for partnum in includeParticle:
kf.write(str(partnum)+"\n")
kf.close()
#get number of particles
numparticles = len(includeParticle)
if excludelist:
self.params['description'] += ( " ... %d particle substack of stackid %d"
% (numparticles, self.params['stackid']))
elif includelist:
self.params['description'] += ( " ... %d particle substack of stackid %d"
% (numparticles, self.params['stackid']))
ogdescr = self.params['description']
for i in range(self.params['split']):
### always do this, if not splitting split=1
sb = os.path.splitext(stackdata['name'])
if self.params['first'] is not None and self.params['last'] is not None:
newname = sb[0]+'.'+str(self.params['first'])+'-'+str(self.params['last'])+sb[-1]
elif self.params['random'] is not None:
newname = "%s-random%d%s"%(sb[0], self.params['random'], sb[-1])
elif self.params['split'] > 1:
fname = 'sub'+str(self.params['stackid'])+'.'+str(i+1)+'.lst'
self.params['keepfile'] = os.path.join(self.params['rundir'],fname)
newname = sb[0]+'.'+str(i+1)+'of'+str(self.params['split'])+sb[-1]
newstack = os.path.join(self.params['rundir'], newname)
apStack.checkForPreviousStack(newstack)
#get number of particles
f = open(self.params['keepfile'], "r")
numparticles = len(f.readlines())
f.close()
self.params['description'] = ogdescr
self.params['description'] += (
(" ... %d particle substack of stackid %d"
% (numparticles, self.params['stackid']))
)
#if splitting, add to description
if self.params['split'] > 1:
self.params['description'] += (" (%i of %i)" % (i+1, self.params['split']))
#create the new sub stack
if not self.params['correctbeamtilt']:
apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=True)
else:
apBeamTilt.makeCorrectionStack(self.params['stackid'], oldstack, newstack)
if not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
apStack.commitSubStack(self.params, newname, sorted=False)
apStack.averageStack(stack=newstack)
newstackid = apStack.getStackIdFromPath(newstack)
if self.params['meanplot'] is True:
apDisplay.printMsg("creating Stack Mean Plot montage for stackid")
apStackMeanPlot.makeStackMeanPlot(newstackid)
def generateParticleParams(self):
paramfile = os.path.join(self.params['rundir'], 'params.iter000.par')
apDisplay.printMsg("Creating parameter file: " + paramfile)
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if os.path.isfile(paramfile):
f = open(paramfile, 'r')
numparam = len(f.readlines())
if numparam > self.params['last']:
apDisplay.printMsg("Param file exists")
return paramfile
else:
apDisplay.printWarning(
"Param file exists with too few particles: %d vs %d" %
(numparam, numpart))
stackdata = apStack.getStackParticlesFromId(self.params['stackid'])
particleparams = {}
f = open(paramfile, 'w')
f.write(
"C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n"
)
apDisplay.printMsg("Writing out particle parameters")
count = 0
t0 = time.time()
self.noeulers = 0
for particle in stackdata:
count += 1
if (count % 200 == 0):
estime = (time.time() - t0) * (numpart - count) / float(count)
apDisplay.printMsg("particle %d -- %s remain" %
(count, apDisplay.timeString(estime)))
if count > self.params['last']:
break
# defaults
## Niko says that if the defocus is negative it will not perform CTF correction
## But it will also not correct the amplitudes
particleparams = {
'ptclnum': particle['particleNumber'],
'psi': 0.0,
'theta': 0.0,
'phi': 0.0,
'df1': -1.0, # workaround if no ctf correction
'df2': -1.0, # set defocus to -1.0 Angstroms
'angast': 0.0,
'mag': 10000, # workaround to get around dstep
'shx': 0.0,
'shy': 0.0,
'film': 1,
'presa': 0.0,
'dpres': 0.0,
}
imagedata = particle['particle']['image']
if self.params['noctf'] is False:
ctfdata, confidence = ctfdb.getBestCtfValueForImage(
imagedata, msg=False, method=self.params['ctfmethod'])
if ctfdata is not None:
### use defocus and astigmatism values
particleparams['df1'] = abs(ctfdata['defocus1'] * 1e10)
particleparams['df2'] = abs(ctfdata['defocus2'] * 1e10)
particleparams['angast'] = ctfdata['angle_astigmatism']
# if using parameters from previous reconstruction
if self.params['reconiterid'] is not None:
emaneuler = self.getStackParticleEulersForIteration(
particle['particleNumber'])
frealigneulers = apFrealign.convertEmanEulersToFrealign(
emaneuler, sym=self.params['sym'])
particleparams['psi'] = frealigneulers['psi']
particleparams['theta'] = frealigneulers['theta']
particleparams['phi'] = frealigneulers['phi']
particleparams['shx'] = emaneuler['shiftx']
particleparams['shy'] = emaneuler['shifty']
if emaneuler['mirror'] is True:
particleparams['shx'] *= -1
self.writeParticleParamLine(particleparams, f)
f.close()
return paramfile
def start(self):
### get stack data
stackdata = apStack.getOnlyStackData(self.params["partstackid"])
###############################
# #
# Andres's refinement steps #
# #
###############################
print "\n"
apDisplay.printMsg("##################################")
apDisplay.printMsg("Starting Andres' refinement steps")
apDisplay.printMsg("##################################")
print "\n"
if self.params["initvol"] is not None:
spidervol = self.convertVolToSpider(mrcvolfile=self.params["initvol"], apix=self.params["initvolapix"])
if self.params["modelid"] is not None:
spidervol = self.convertVolToSpider(modelid=self.params["modelid"])
partstack = stackdata["path"]["path"] + "/" + stackdata["name"]
spiderstack = self.convertStackToSpider(partstack)
partsnum = apStack.getNumberStackParticlesFromId(self.params["partstackid"])
for j in range(self.params["refineiters"]):
iternum = j + 1
appiondata.ApPathData.direct_query(1)
apDisplay.printMsg("Starting projection-matching refinement/XMIPP iteration " + str(iternum))
boxsize = self.getBoxSize()
### projection-matching refinement/XMIPP
apshout, apshstack, apsheuler, projstack, numprojs = self.projMatchRefine(
spidervol, spiderstack, boxsize, partsnum, self.params["radius"], iternum
)
apDisplay.printMsg("Calculating weighted cross-correlation coefficients")
### calculation of weighted cross-correlation coefficients
apshout_weighted = self.cccAPSH(apshout, iternum)
apDisplay.printMsg("Creating select files based on weighted cross-correlation coefficients")
### create select files based on calculated weighted-cross-correlation
corrSelect = self.makecccAPSHselectFile(apshout_weighted, iternum, factor=0.1)
### create volume file names
apshVolfile = os.path.join(self.params["rundir"], "apshVolume-BPCG-%03d.spi" % (iternum))
apshVolfile2 = os.path.join(self.params["rundir"], "apshVolume-BPRP-%03d.spi" % (iternum))
### run BPRP on selected particles
backprojectPWL.backprojectCG(apshstack, apsheuler, apshVolfile, partsnum, self.params["radius"])
self.APSHbackProject(apshstack, apsheuler, apshVolfile2, partsnum)
### center volume
filename = os.path.splitext(apshVolfile)[0]
apshVolFileCentered = filename + "_centered.spi"
backprojectPWL.centerVolume(apshVolfile, apshVolFileCentered)
### generate class averages
backprojectPWL.createClassAverages(
apshstack,
projstack,
apshout,
numprojs,
boxsize,
outclass="classes",
rotated=True,
shifted=True,
dataext=".spi",
)
print "check~~!!!"
sys.exit(1)
### calculate FSC
### generate odd and even select files for FSC calculation
corrSelectOdd, corrSelectEven = self.splitOddEven(cnum, corrSelect, iternum)
fscout = self.runEoTest(corrSelectOdd, corrSelectEven, cnum, apshstack, apsheuler, iternum)
self.runRmeasure(apshVolFileCentered)
### filter volume
backproject.butterworthFscLP(apshVolFileCentered, fscout)
### reset file names for next round
volfile = apshVolFileCentered
eulerfile = apsheuler
mrcvolfile = self.processVolume(volfile, cnum, iternum)
print "\n"
apDisplay.printMsg("###########################")
apDisplay.printMsg("Done with iteration " + str(j + 1) + "")
apDisplay.printMsg("###########################")
print "\n"
# if len(self.classlist) > 1:
# get a list of all unique combinations of volumes
# pairlist = self.computeClassVolPair()
### insert volumes into DB
self.insertOtrRun(mrcvolfile)
def start(self):
### get stack data
stackdata = apStack.getOnlyStackData(self.params['partstackid'])
###############################
# #
# Andres's refinement steps #
# #
###############################
print "\n"
apDisplay.printMsg("##################################")
apDisplay.printMsg("Starting Andres' refinement steps")
apDisplay.printMsg("##################################")
print "\n"
if self.params['initvol'] is not None:
spidervol = self.convertVolToSpider(
mrcvolfile=self.params['initvol'],
apix=self.params['initvolapix'])
if self.params['modelid'] is not None:
spidervol = self.convertVolToSpider(modelid=self.params['modelid'])
partstack = stackdata['path']['path'] + "/" + stackdata['name']
spiderstack = self.convertStackToSpider(partstack)
partsnum = apStack.getNumberStackParticlesFromId(
self.params['partstackid'])
for j in range(self.params['refineiters']):
iternum = j + 1
appiondata.ApPathData.direct_query(1)
apDisplay.printMsg(
"Starting projection-matching refinement/XMIPP iteration " +
str(iternum))
boxsize = self.getBoxSize()
### projection-matching refinement/XMIPP
apshout, apshstack, apsheuler, projstack, numprojs = self.projMatchRefine(
spidervol, spiderstack, boxsize, partsnum,
self.params['radius'], iternum)
apDisplay.printMsg(
"Calculating weighted cross-correlation coefficients")
### calculation of weighted cross-correlation coefficients
apshout_weighted = self.cccAPSH(apshout, iternum)
apDisplay.printMsg(
"Creating select files based on weighted cross-correlation coefficients"
)
### create select files based on calculated weighted-cross-correlation
corrSelect = self.makecccAPSHselectFile(apshout_weighted,
iternum,
factor=0.1)
### create volume file names
apshVolfile = os.path.join(self.params['rundir'],
"apshVolume-BPCG-%03d.spi" % (iternum))
apshVolfile2 = os.path.join(self.params['rundir'],
"apshVolume-BPRP-%03d.spi" % (iternum))
### run BPRP on selected particles
backprojectPWL.backprojectCG(apshstack, apsheuler, apshVolfile,
partsnum, self.params['radius'])
self.APSHbackProject(apshstack, apsheuler, apshVolfile2, partsnum)
### center volume
filename = os.path.splitext(apshVolfile)[0]
apshVolFileCentered = filename + "_centered.spi"
backprojectPWL.centerVolume(apshVolfile, apshVolFileCentered)
### generate class averages
backprojectPWL.createClassAverages(apshstack,
projstack,
apshout,
numprojs,
boxsize,
outclass="classes",
rotated=True,
shifted=True,
dataext=".spi")
print "check~~!!!"
sys.exit(1)
### calculate FSC
### generate odd and even select files for FSC calculation
corrSelectOdd, corrSelectEven = self.splitOddEven(
cnum, corrSelect, iternum)
fscout = self.runEoTest(corrSelectOdd, corrSelectEven, cnum,
apshstack, apsheuler, iternum)
self.runRmeasure(apshVolFileCentered)
### filter volume
backproject.butterworthFscLP(apshVolFileCentered, fscout)
### reset file names for next round
volfile = apshVolFileCentered
eulerfile = apsheuler
mrcvolfile = self.processVolume(volfile, cnum, iternum)
print "\n"
apDisplay.printMsg("###########################")
apDisplay.printMsg("Done with iteration " + str(j + 1) + "")
apDisplay.printMsg("###########################")
print "\n"
#if len(self.classlist) > 1:
#get a list of all unique combinations of volumes
# pairlist = self.computeClassVolPair()
### insert volumes into DB
self.insertOtrRun(mrcvolfile)
def generateParticleParams(self):
paramfile = os.path.join(self.params["rundir"], "params.iter000.par")
apDisplay.printMsg("Creating parameter file: " + paramfile)
numpart = apStack.getNumberStackParticlesFromId(self.params["stackid"])
if os.path.isfile(paramfile):
f = open(paramfile, "r")
numparam = len(f.readlines())
if numparam > self.params["last"]:
apDisplay.printMsg("Param file exists")
return paramfile
else:
apDisplay.printWarning("Param file exists with too few particles: %d vs %d" % (numparam, numpart))
stackdata = apStack.getStackParticlesFromId(self.params["stackid"])
particleparams = {}
f = open(paramfile, "w")
f.write("C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n")
apDisplay.printMsg("Writing out particle parameters")
count = 0
t0 = time.time()
self.noeulers = 0
for particle in stackdata:
count += 1
if count % 200 == 0:
estime = (time.time() - t0) * (numpart - count) / float(count)
apDisplay.printMsg("particle %d -- %s remain" % (count, apDisplay.timeString(estime)))
if count > self.params["last"]:
break
# defaults
## Niko says that if the defocus is negative it will not perform CTF correction
## But it will also not correct the amplitudes
particleparams = {
"ptclnum": particle["particleNumber"],
"psi": 0.0,
"theta": 0.0,
"phi": 0.0,
"df1": -1.0, # workaround if no ctf correction
"df2": -1.0, # set defocus to -1.0 Angstroms
"angast": 0.0,
"mag": 10000, # workaround to get around dstep
"shx": 0.0,
"shy": 0.0,
"film": 1,
"presa": 0.0,
"dpres": 0.0,
}
imagedata = particle["particle"]["image"]
if self.params["noctf"] is False:
ctfdata, confidence = ctfdb.getBestCtfValueForImage(
imagedata, msg=False, method=self.params["ctfmethod"]
)
if ctfdata is not None:
### use defocus and astigmatism values
particleparams["df1"] = abs(ctfdata["defocus1"] * 1e10)
particleparams["df2"] = abs(ctfdata["defocus2"] * 1e10)
particleparams["angast"] = ctfdata["angle_astigmatism"]
# if using parameters from previous reconstruction
if self.params["reconiterid"] is not None:
emaneuler = self.getStackParticleEulersForIteration(particle["particleNumber"])
frealigneulers = apFrealign.convertEmanEulersToFrealign(emaneuler, sym=self.params["sym"])
particleparams["psi"] = frealigneulers["psi"]
particleparams["theta"] = frealigneulers["theta"]
particleparams["phi"] = frealigneulers["phi"]
particleparams["shx"] = emaneuler["shiftx"]
particleparams["shy"] = emaneuler["shifty"]
if emaneuler["mirror"] is True:
particleparams["shx"] *= -1
self.writeParticleParamLine(particleparams, f)
f.close()
return paramfile
def generateParticleParams(self):
paramfile = os.path.join(self.params['rundir'], 'params.iter000.par')
apDisplay.printMsg("Creating parameter file: "+paramfile)
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if os.path.isfile(paramfile):
f = open(paramfile, 'r')
numparam = len(f.readlines())
if numparam > self.params['last']:
apDisplay.printMsg("Param file exists")
return paramfile
else:
apDisplay.printWarning("Param file exists with too few particles: %d vs %d"%(numparam,numpart))
stackdata = apStack.getStackParticlesFromId(self.params['stackid'])
particleparams={}
f = open(paramfile, 'w')
f.write("C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n")
apDisplay.printMsg("Writing out particle parameters")
count = 0
t0 = time.time()
self.noeulers = 0
for particle in stackdata:
count += 1
if (count % 200 == 0):
estime = (time.time() - t0) * (numpart-count) / float(count)
apDisplay.printMsg("particle %d -- %s remain"%(count, apDisplay.timeString(estime)))
if count > self.params['last']:
break
# defaults
## Niko says that if the defocus is negative it will not perform CTF correction
## But it will also not correct the amplitudes
particleparams = {
'ptclnum': particle['particleNumber'],
'psi': 0.0,
'theta': 0.0,
'phi': 0.0,
'df1': -1.0, # workaround if no ctf correction
'df2': -1.0, # set defocus to -1.0 Angstroms
'angast': 0.0,
'mag': 10000, # workaround to get around dstep
'shx': 0.0,
'shy': 0.0,
'film': 1,
'presa': 0.0,
'dpres': 0.0,
}
imagedata = particle['particle']['image']
if self.params['noctf'] is False:
ctfdata, confidence = ctfdb.getBestCtfValueForImage(imagedata, msg=False, method=self.params['ctfmethod'])
if ctfdata is not None:
### use defocus and astigmatism values
particleparams['df1'] = abs(ctfdata['defocus1']*1e10)
particleparams['df2'] = abs(ctfdata['defocus2']*1e10)
particleparams['angast'] = ctfdata['angle_astigmatism']
# if using parameters from previous reconstruction
if self.params['reconiterid'] is not None:
emaneuler = self.getStackParticleEulersForIteration(particle['particleNumber'])
frealigneulers = apFrealign.convertEmanEulersToFrealign(emaneuler, sym=self.params['sym'])
particleparams['psi'] = frealigneulers['psi']
particleparams['theta'] = frealigneulers['theta']
particleparams['phi'] = frealigneulers['phi']
particleparams['shx'] = emaneuler['shiftx']
particleparams['shy'] = emaneuler['shifty']
if emaneuler['mirror'] is True:
particleparams['shx'] *= -1
self.writeParticleParamLine(particleparams,f)
f.close()
return paramfile
def start(self):
#old stack size
stacksize = apStack.getNumberStackParticlesFromId(
self.params['stackid'])
# if exclude or include lists are not defined...
if self.params['exclude'] is None and self.params['include'] is None:
# if first and last are specified, create a file
if self.params['first'] is not None and self.params[
'last'] is not None:
stp = str(self.params['first'])
enp = str(self.params['last'])
fname = 'sub' + str(
self.params['stackid']) + '_' + stp + '-' + enp + '.lst'
self.params['keepfile'] = os.path.join(self.params['rundir'],
fname)
apDisplay.printMsg("Creating keep list: " +
self.params['keepfile'])
f = open(self.params['keepfile'], 'w')
for i in range(self.params['first'], self.params['last'] + 1):
f.write('%d\n' % (int(i) - 1))
f.close()
# generate the random list by giving number and create the file
elif self.params['random'] is not None:
#numOfRandomParticles = str(self.params['random'])
#fname = 'random'+str(self.params['stackid'])+'_'+numOfRandomParticles+'.lst'
fname = "random%d_%d.lst" % (self.params['stackid'],
self.params['random'])
self.params['keepfile'] = os.path.join(self.params['rundir'],
fname)
apDisplay.printMsg("Creating keep list: " +
self.params['keepfile'])
# create a file
f = open(self.params['keepfile'], 'w')
# generate a random sequence by giving size
randomList = random.sample(xrange(self.params['last']),
self.params['random'])
randomList.sort()
for partnum in randomList:
f.write('%d\n' % partnum)
f.close()
# if splitting, create files containing the split values
elif self.params['split'] > 1:
for i in range(self.params['split']):
fname = 'sub' + str(
self.params['stackid']) + '.' + str(i + 1) + '.lst'
self.params['keepfile'] = os.path.join(
self.params['rundir'], fname)
apDisplay.printMsg("Creating keep list: " +
self.params['keepfile'])
f = open(self.params['keepfile'], 'w')
for p in range(stacksize):
if (p % self.params['split']) - i == 0:
f.write('%i\n' % p)
f.close()
# if exclude-from option is specified, convert particles to exclude
elif self.params['excludefile'] is True:
oldkf = open(self.params['keepfile'])
partlist = []
for line in oldkf:
particle = line.strip()
try:
particle = int(particle)
except:
continue
partlist.append(particle)
oldkf.close()
# create new list excluding the particles
apDisplay.printMsg("Converting keep file to exclude file")
newkeepfile = "tmpnewkeepfile.txt"
newkf = open(newkeepfile, 'w')
for p in range(stacksize):
if p not in partlist:
newkf.write("%i\n" % p)
newkf.close()
self.params['keepfile'] = os.path.abspath(newkeepfile)
# otherwise, just copy the file
elif not os.path.isfile(os.path.basename(self.params['keepfile'])):
shutil.copy(self.params['keepfile'],
os.path.basename(self.params['keepfile']))
# if either exclude or include lists is defined
elif self.params['exclude'] or self.params['include']:
### list of particles to be excluded
excludelist = []
if self.params['exclude'] is not None:
excludestrlist = self.params['exclude'].split(",")
for excld in excludestrlist:
excludelist.append(int(excld.strip()))
apDisplay.printMsg("Exclude list: " + str(excludelist))
### list of particles to be included
includelist = []
if self.params['include'] is not None:
includestrlist = self.params['include'].split(",")
for incld in includestrlist:
includelist.append(int(incld.strip()))
apDisplay.printMsg("Include list: " + str(includelist))
#new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
newname = stackdata['name']
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
#if include or exclude list is given...
if self.params['include'] is not None or self.params[
'exclude'] is not None:
includeParticle = []
excludeParticle = 0
for partnum in range(stacksize):
if includelist and partnum in includelist:
includeParticle.append(partnum)
elif excludelist and not partnum in excludelist:
includeParticle.append(partnum)
else:
excludeParticle += 1
includeParticle.sort()
### write kept particles to file
self.params['keepfile'] = os.path.join(
self.params['rundir'], "keepfile-" + self.timestamp + ".list")
apDisplay.printMsg("writing to keepfile " +
self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
for partnum in includeParticle:
kf.write(str(partnum) + "\n")
kf.close()
#get number of particles
numparticles = len(includeParticle)
if excludelist:
self.params['description'] += (
" ... %d particle substack of stackid %d" %
(numparticles, self.params['stackid']))
elif includelist:
self.params['description'] += (
" ... %d particle substack of stackid %d" %
(numparticles, self.params['stackid']))
ogdescr = self.params['description']
for i in range(self.params['split']):
### always do this, if not splitting split=1
sb = os.path.splitext(stackdata['name'])
if self.params['first'] is not None and self.params[
'last'] is not None:
newname = sb[0] + '.' + str(self.params['first']) + '-' + str(
self.params['last']) + sb[-1]
elif self.params['random'] is not None:
newname = "%s-random%d%s" % (sb[0], self.params['random'],
sb[-1])
elif self.params['split'] > 1:
fname = 'sub' + str(
self.params['stackid']) + '.' + str(i + 1) + '.lst'
self.params['keepfile'] = os.path.join(self.params['rundir'],
fname)
newname = sb[0] + '.' + str(i + 1) + 'of' + str(
self.params['split']) + sb[-1]
newstack = os.path.join(self.params['rundir'], newname)
apStack.checkForPreviousStack(newstack)
#get number of particles
f = open(self.params['keepfile'], "r")
numparticles = len(f.readlines())
f.close()
self.params['description'] = ogdescr
self.params['description'] += (
(" ... %d particle substack of stackid %d" %
(numparticles, self.params['stackid'])))
#if splitting, add to description
if self.params['split'] > 1:
self.params['description'] += (" (%i of %i)" %
(i + 1, self.params['split']))
#create the new sub stack
if not self.params['correctbeamtilt']:
apStack.makeNewStack(oldstack,
newstack,
self.params['keepfile'],
bad=True)
else:
apBeamTilt.makeCorrectionStack(self.params['stackid'],
oldstack, newstack)
if not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
apStack.commitSubStack(self.params, newname, sorted=False)
apStack.averageStack(stack=newstack)
newstackid = apStack.getStackIdFromPath(newstack)
if self.params['meanplot'] is True:
apDisplay.printMsg(
"creating Stack Mean Plot montage for stackid")
apStackMeanPlot.makeStackMeanPlot(newstackid)
