def getResolutionData(self, avgpath, iternum):
evenvol = os.path.join(avgpath, "avglist3_%dp1.mrc" % (self.params["rescut"]))
oddvol = os.path.join(avgpath, "avglist3_%dp2.mrc" % (self.params["rescut"]))
evenhed = os.path.join(avgpath, "avglist3_%dp1.hed" % (self.params["rescut"]))
oddhed = os.path.join(avgpath, "avglist3_%dp2.hed" % (self.params["rescut"]))
emancmd1 = "proc3d %s %s" % (evenvol, evenhed)
apEMAN.executeEmanCmd(emancmd1, verbose=True)
emancmd2 = "proc3d %s %s" % (oddvol, oddhed)
apEMAN.executeEmanCmd(emancmd2, verbose=True)
fscfile = "fsc.eotest.%d" % (iternum)
fscpath = os.path.join(avgpath, fscfile)
emancmd3 = "proc3d %s %s fsc=%s" % (evenhed, oddhed, fscpath)
apEMAN.executeEmanCmd(emancmd3, verbose=True)
if not os.path.isfile(fscpath):
apDisplay.printWarning("Could not find FSC file: " + fscpath)
return None
f = open(fscpath, "r")
xy = f.readlines()
lines = len(xy)
boxsize = lines * 2.0
f.close()
# calculate the resolution:
halfres = apRecon.calcRes(fscpath, boxsize, self.params["step"])
apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms" % (halfres), "cyan")
# save to database
resq = appiondata.ApResolutionData()
resq["half"] = halfres
resq["fscfile"] = fscpath
return resq
def getResolutionData(self, avgpath, iternum):
evenvol = os.path.join(avgpath, "avglist3_%dp1.mrc"%(self.params['rescut']))
oddvol = os.path.join(avgpath, "avglist3_%dp2.mrc"%(self.params['rescut']))
evenhed = os.path.join(avgpath, "avglist3_%dp1.hed"%(self.params['rescut']))
oddhed = os.path.join(avgpath, "avglist3_%dp2.hed"%(self.params['rescut']))
emancmd1 = 'proc3d %s %s' % (evenvol, evenhed)
apEMAN.executeEmanCmd(emancmd1, verbose=True)
emancmd2 = 'proc3d %s %s' % (oddvol, oddhed)
apEMAN.executeEmanCmd(emancmd2, verbose=True)
fscfile = 'fsc.eotest.%d'%(iternum)
fscpath = os.path.join(avgpath, fscfile)
emancmd3 = 'proc3d %s %s fsc=%s' % (evenhed, oddhed, fscpath)
apEMAN.executeEmanCmd(emancmd3, verbose=True)
if not os.path.isfile(fscpath):
apDisplay.printWarning("Could not find FSC file: "+fscpath)
return None
f = open(fscpath, 'r')
xy = f.readlines()
lines = len(xy)
boxsize = (lines * 2.0)
f.close()
# calculate the resolution:
halfres = apRecon.calcRes(fscpath, boxsize, self.params['step'])
apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms"%(halfres), "cyan")
# save to database
resq=appiondata.ApResolutionData()
resq['half'] = halfres
resq['fscfile'] = fscpath
return resq
def getResolutionData(self, iternum):
fscfile = 'fsc.eotest.%d'%(iternum)
fscpath = os.path.join(self.params['rundir'], "iter%03d"%(iternum), fscfile)
if not os.path.isfile(fscpath):
apDisplay.printWarning("Could not find FSC file: "+fscpath)
return None
# calculate the resolution:
halfres = apRecon.calcRes(fscpath, self.boxsize, self.apix)
apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms"%(halfres), "cyan")
# save to database
resq=appiondata.ApResolutionData()
resq['half'] = halfres
resq['fscfile'] = fscfile
return resq
def getResolutionData(self, iternum):
fscfile = 'fsc.eotest.%d'%(iternum)
fscpath = os.path.join(self.params['rundir'], "iter%03d"%(iternum), fscfile)
if not os.path.isfile(fscpath):
apDisplay.printWarning("Could not find FSC file: "+fscpath)
return None
# calculate the resolution:
halfres = apRecon.calcRes(fscpath, self.boxsize, self.apix)
apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms"%(halfres), "cyan")
# save to database
resq=appiondata.ApResolutionData()
resq['half'] = halfres
resq['fscfile'] = fscfile
return resq
def getResolutionData(self, iteration):
fsc = 'fsc.eotest.'+iteration['num']
fscfile = os.path.join(self.params['rundir'],fsc)
if not os.path.isfile(fscfile):
apDisplay.printWarning("Could not find FSC file: "+fscfile)
iteration['fscfile'] = fscfile
if not fsc in self.params['fscs']:
return None
# calculate the resolution:
halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix'])
# save to database
resq=appiondata.ApResolutionData()
resq['half'] = halfres
resq['fscfile'] = fsc
return resq
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
# symmetry info
if self.params['sym']=='Icos':
self.params['symtype']='I'
self.params['symfold']=None
else:
self.params['symtype']=self.params['sym'][0]
self.params['symfold']=int(self.params['sym'][1:])
# eman "d" symmetry is spider "ci"
if self.params['symtype'].upper()=="D":
self.params['symtype'] = "CI"
# create symmetry doc file
sydoc="sym.spi"
refine.symmetryDoc(self.params['symtype'],self.params['symfold'],sydoc)
# convert incr to array of increments
ang_inc=self.params['incr'].split(',')
self.params['numiter'] = len(ang_inc)
self.params['increments']=[]
for i in range(0,self.params['numiter']):
self.params['increments'].append(int(ang_inc[i]))
# convert stack to spider stack
spiderstack=os.path.join(self.params['rundir'],'start.spi')
operations.stackToSpiderStack(
self.stack['file'],
spiderstack,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
numpart=self.params['numpart']
)
# create filtered stack
spiderstackfilt=spiderstack
if (self.params['lowpass']+self.params['highpass']) > 0:
spiderstackfilt=os.path.join(self.params['rundir'],'start_filt.spi')
operations.stackToSpiderStack(
self.stack['file'],
spiderstackfilt,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
lp=self.params['lowpass'],
hp=self.params['highpass'],
numpart=self.params['numpart']
)
# rescale initial model if necessary
outvol = os.path.join(self.params['rundir'],"vol000.spi")
apModel.rescaleModel(self.params['modelid'], outvol, self.stack['boxsize'], self.stack['apix'], spider=True)
self.params['itervol']=outvol
for iter in range(1,self.params['numiter']+1):
# create projections for projection matching
apDisplay.printMsg("creating reference projections of volume: %s" % self.params['itervol'])
projs,numprojs,ang,sel = refine.createProjections(
incr=self.params['increments'][iter-1],
boxsz=self.stack['boxsize'],
symfold=self.params['symfold'],
invol=self.params['itervol'],
rad=self.params['rad'],
)
# run reference-based alignment
apDisplay.printMsg("running reference-based alignment (AP MQ)")
apmqfile = "apmq%03d.spi" % iter
outang = "angular%03d.spi" % iter
shf = "shifts%03d.spi" % iter
shiftedStack="parts_shifted.spi"
apFile.removeFile(shf)
refine.spiderAPMQ(
projs=projs,
numprojs=numprojs,
tsearch=self.params['xysearch'],
tstep=self.params['xystep'],
firstRing=self.params['firstring'],
lastRing=self.params['lastring'],
stackfile=spiderstackfilt,
nump=self.params['numpart'],
ang=ang,
apmqfile=apmqfile,
outang=outang,
nproc=self.params['proc'],
)
# use cross-correlation to find the sub-pixel alignment
# of the particles,
# results will be saved in "peakfile.spi"
apFile.removeFile(shiftedStack)
# don't use MPI here - for some reason slower?
mySpi=spyder.SpiderSession(dataext=".spi", logo=False, log=False)
apmqlist = refine.readDocFile(apmqfile)
avgccrot = 0
apDisplay.printMsg("creating shifted stack")
for p in range(0,self.params['numpart']):
ref=int(float(apmqlist[p][2]))
ccrot=float(apmqlist[p][3])
inplane=float(apmqlist[p][4])
avgccrot+=ccrot
# invert the sign - ref projs will be rotated
inplane*=-1
# get corresponding projection
if (ref <= 0):
# mirror projection if necessary
ref*=-1
refimg=spyder.fileFilter(projs)+"@"+str(ref)
refine.mirrorImg(refimg,"_3",inMySpi=mySpi)
img="_3"
else:
img=spyder.fileFilter(projs)+"@"+str(ref)
refine.rotAndShiftImg(img,"_2",inplane,inMySpi=mySpi)
refine.maskImg("_2","_3",self.params['rad'],"D","E",
center=int((self.stack['boxsize']/2)+1),
inMySpi=mySpi)
# pad ref image & stack image to twice the size
refine.padImg("_3","_2",2*self.stack['boxsize'],"N",1,1,0,inMySpi=mySpi)
stackimg=spyder.fileFilter(spiderstack)+"@"+str(p+1)
refine.padImg(stackimg,"_1",2*self.stack['boxsize'],"B",1,1,inMySpi=mySpi)
# calculate cross-correlation
refine.getCC("_1","_2","_1",inMySpi=mySpi)
# crop the correllation image to allowable shift amount
shift=int(self.params['allowedShift']*self.stack['boxsize'])
dim=2*shift+1
topleftx=self.stack['boxsize']-shift+1
refine.windowImg("_1","_2",dim,topleftx,topleftx,inMySpi=mySpi)
# find the sub-pixel location of cc peak
mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17","_2","0")
# create new stack of shifted particles
shpos=spyder.fileFilter(shiftedStack)+"@"+str(p+1)
mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN")
mySpi.toSpiderQuiet("GP x17","_2",str(shift+1)+","+str(shift+1))
refine.copyImg(stackimg,shpos,inMySpi=mySpi)
mySpi.toSpiderQuiet("ELSE")
#mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16")
mySpi.toSpiderQuiet("SH F",stackimg,shpos,"-x15,-x16")
mySpi.toSpiderQuiet("ENDIF")
# save shifts to file
mySpi.toSpiderQuiet("SD "+str(p+1)+",x15,x16,x17",spyder.fileFilter(shf))
mySpi.toSpiderQuiet("SD E",spyder.fileFilter(shf))
mySpi.close()
# create class average images
refine.createClassAverages(
shiftedStack,
projs,
apmqfile,
numprojs,
self.stack['boxsize'],
shifted=True,
)
# rename class averages & variacnes for iteration
cmd="/bin/mv classes.hed classes.%d.hed;" % iter
cmd+="/bin/mv classes.img classes.%d.img;" % iter
cmd+="/bin/mv variances.hed variances.%d.hed;" % iter
cmd+="/bin/mv variances.img variances.%d.img;" % iter
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
# calculate the stddev for the apmq cc's for throwing out particles
avgccrot/=self.params['numpart']
stdccrot = 0
for p in range(0,self.params['numpart']):
stdccrot+=abs(float(apmqlist[p][3])-avgccrot)
stdccrot/=self.params['numpart']
cccutoff=avgccrot+(stdccrot*self.params['keepsig'])
apDisplay.printMsg("average cc: %f" %avgccrot)
apDisplay.printMsg("setting cutoff to: %f" %cccutoff)
# create new selection file that only has particles with good cc's
selectfile="select%03d.spi" % iter
apFile.removeFile(selectfile)
mySpi = spyder.SpiderSession(nproc=self.params['proc'],dataext=".spi", logo=False, log=False)
i=1
for p in range(0,self.params['numpart']):
ccrot=float(apmqlist[p][3])
if ccrot>=cccutoff:
mySpi.toSpiderQuiet("x11=%d" % (p+1))
mySpi.toSpiderQuiet("SD %d,x11" % i,spyder.fileFilter(selectfile))
i+=1
mySpi.close()
# calculate the new 3d structure using centered projections
# and the corrected angles from the angular doc file
apDisplay.printMsg("creating 3d volume")
out_rawvol="vol_raw%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
selectfile,
rad=self.params['rad'],
ang=outang,
out=out_rawvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['bpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'],
)
else:
refine.backProjection(
shiftedStack,
selectfile,
ang=outang,
out=out_rawvol,
sym=sydoc,
nproc=self.params['proc']
)
# create even & odd select files
apDisplay.printMsg("creating even/odd volumes")
oddfile="selectodd%03d.spi" % iter
evenfile="selecteven%03d.spi" % iter
refine.docSplit(selectfile,oddfile,evenfile)
# get the even & odd volumesa
oddvol="vol1%03d.spi" % iter
evenvol="vol2%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
oddfile,
rad=self.params['rad'],
ang=outang,
out=oddvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc']
)
refine.iterativeBackProjection(
shiftedStack,
evenfile,
rad=self.params['rad'],
ang=outang,
out=evenvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc']
)
else:
refine.backProjection(
shiftedStack,
oddfile,
ang=outang,
out=oddvol,
sym=sydoc,
nproc=self.params['proc'],
)
refine.backProjection(
shiftedStack,
evenfile,
ang=outang,
out=evenvol,
sym=sydoc,
nproc=self.params['proc'],
)
# calculate the FSC
apDisplay.printMsg("calculating FSC")
fscfile="fsc%03d.spi" % iter
emanfsc="fsc.eotest.%d" % iter
refine.calcFSC(oddvol,evenvol,fscfile)
# convert to eman-style fscfile
refine.spiderFSCtoEMAN(fscfile,emanfsc)
# calculate the resolution at 0.5 FSC & write to file
res=apRecon.calcRes(emanfsc,self.stack['boxsize'],self.stack['apix'])
restxt="resolution.txt"
if iter==1 and os.path.isfile(restxt):
os.remove(restxt)
resfile=open(restxt,"a")
resfile.write("iter %d:\t%.3f\n" % (iter,res))
resfile.close()
# filter & normalize the volume to be used as a reference in the next round
outvol="vol%03d.spi" % iter
emancmd="proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % (out_rawvol,outvol,self.stack['apix'],res,self.params['rad'])
if self.params['imask'] is not None:
emancmd+=" imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# create mrc files of volumes
emancmd="proc3d %s %s apix=%.3f mask=%d norm" % (out_rawvol,"threed.%da.mrc" % iter, self.stack['apix'],self.params['rad'])
if self.params['imask'] is not None:
emancmd+=" imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
emancmd="proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % (out_rawvol,"threed.%da.lp.mrc" % iter, self.stack['apix'],res, self.params['rad'])
if self.params['imask'] is not None:
emancmd+=" imask=%d" % self.param['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# set this model as start for next iteration, remove previous
os.remove(self.params['itervol'])
os.remove(out_rawvol)
self.params['itervol']=outvol
# clean up directory
apDisplay.printMsg("cleaning up directory")
if os.path.isfile(oddvol):
os.remove(oddvol)
if os.path.isfile(evenvol):
os.remove(evenvol)
if os.path.isfile(ang):
os.remove(ang)
if os.path.isfile(sel):
os.remove(sel)
if os.path.isfile(projs):
os.remove(projs)
if os.path.isfile(oddfile):
os.remove(oddfile)
if os.path.isfile(evenfile):
os.remove(evenfile)
if os.path.isfile(emanfsc) and os.path.isfile(fscfile):
os.remove(fscfile)
if os.path.isfile(shiftedStack):
os.remove(shiftedStack)
os.remove(self.params['itervol'])
def insertIteration(self, iteration):
refineparamsq = appiondata.ApEmanRefineIterData()
refineparamsq['ang'] = iteration['ang']
refineparamsq['lpfilter'] = iteration['lpfilter']
refineparamsq['hpfilter'] = iteration['hpfilter']
refineparamsq['pad'] = iteration['pad']
refineparamsq['EMAN_maxshift'] = iteration['maxshift']
refineparamsq['EMAN_hard'] = iteration['hard']
refineparamsq['EMAN_classkeep'] = iteration['classkeep']
refineparamsq['EMAN_classiter'] = iteration['classiter']
refineparamsq['EMAN_filt3d'] = iteration['filt3d']
refineparamsq['EMAN_shrink'] = iteration['shrink']
refineparamsq['EMAN_euler2'] = iteration['euler2']
refineparamsq['EMAN_xfiles'] = iteration['xfiles']
refineparamsq['EMAN_median'] = iteration['median']
refineparamsq['EMAN_phasecls'] = iteration['phasecls']
refineparamsq['EMAN_fscls'] = iteration['fscls']
refineparamsq['EMAN_refine'] = iteration['refine']
refineparamsq['EMAN_goodbad'] = iteration['goodbad']
refineparamsq['EMAN_perturb'] = iteration['perturb']
refineparamsq['MsgP_cckeep'] = iteration['msgpasskeep']
refineparamsq['MsgP_minptls'] = iteration['msgpassminp']
#create Chimera snapshots
fscfile = os.path.join(self.params['rundir'],
"fsc.eotest." + iteration['num'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'],
self.params['apix'])
if self.params['snapfilter']:
halfres = self.params['snapfilter']
volumeDensity = 'threed.' + iteration['num'] + 'a.mrc'
volDensPath = os.path.join(self.params['rundir'], volumeDensity)
apChimera.filterAndChimera(volDensPath,
halfres,
self.params['apix'],
self.params['boxsize'],
'snapshot',
self.params['contour'],
self.params['zoom'],
sym=iteration['sym']['eman_name'],
mass=self.params['mass'])
## uncommment this for chimera image only runs...
if self.params['chimeraonly'] is True:
return
# insert resolution data
if halfres != True:
resData = self.getResolutionData(iteration)
else:
apDisplay.printWarning(
"resolution reported as nan, not committing results to database"
)
return
if self.params['package'] == 'EMAN':
refineclassavg = 'classes_eman.' + iteration['num'] + '.img'
postrefineclassavg = None
elif self.params['package'] == 'EMAN/SpiCoran':
refineclassavg = 'classes_eman.' + iteration['num'] + '.img'
postrefineclassavg = 'classes_coran.' + iteration['num'] + '.img'
elif self.params['package'] == 'EMAN/MsgP':
refineclassavg = 'classes_eman.' + iteration['num'] + '.img'
postrefineclassavg = 'classes_msgp.' + iteration['num'] + '.img'
else:
apDisplay.printError("Refinement Package Not Valid")
# insert refinement results
refineq = appiondata.ApRefineIterData()
refineq['refineRun'] = self.params['refineRun']
refineq['emanParams'] = refineparamsq
refineq['iteration'] = iteration['num']
refineq['resolution'] = resData
refineq['rMeasure'] = self.getRMeasureData(iteration)
refineq['mask'] = iteration['mask']
refineq['imask'] = iteration['imask']
refineq['symmetry'] = iteration['sym']
refineq['exemplar'] = False
classvar = 'classes.' + iteration['num'] + '.var.img'
refineq['refineClassAverages'] = refineclassavg
refineq['postRefineClassAverages'] = postrefineclassavg
if classvar in self.params['classvars']:
refineq['classVariance'] = classvar
if volumeDensity in self.params['volumes']:
refineq['volumeDensity'] = volumeDensity
apDisplay.printMsg("inserting Refinement Data into database")
if self.params['commit'] is True:
refineq.insert()
else:
apDisplay.printWarning("not committing results to database")
#insert FSC data
fscfile = os.path.join(self.params['rundir'],
"fsc.eotest." + iteration['num'])
self.insertFSC(fscfile, refineq, self.params['commit'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'],
self.params['apix'])
apDisplay.printColor("FSC 0.5 Resolution: " + str(halfres), "cyan")
# get projections eulers for iteration:
eulers = self.getEulersFromProj(iteration['num'])
# get list of bad particles for this iteration
badprtls = self.readParticleLog(self.params['rundir'],
iteration['num'])
# expand cls.*.tar into temp file
clsf = os.path.join(self.params['rundir'],
"cls." + iteration['num'] + ".tar")
#print "reading",clsf
clstar = tarfile.open(clsf)
clslist = clstar.getmembers()
clsnames = clstar.getnames()
#print "extracting",clsf,"into temp directory"
for clsfile in clslist:
clstar.extract(clsfile, self.params['tmpdir'])
clstar.close()
# for each class, insert particle alignment info into database
apDisplay.printColor(
"Inserting Particle Classification Data for " +
str(len(clsnames)) + " classes", "magenta")
t0 = time.time()
for cls in clsnames:
self.insertRefineParticleData(cls, iteration, eulers, badprtls,
refineq, len(clsnames))
apDisplay.printColor(
"\nFinished in " + apDisplay.timeString(time.time() - t0),
"magenta")
# remove temp directory
for file in os.listdir(self.params['tmpdir']):
os.remove(os.path.join(self.params['tmpdir'], file))
os.rmdir(self.params['tmpdir'])
#create euler freq map
apDisplay.printMsg("creating euler frequency map")
refrunid = int(self.params['refineRun'].dbid)
iternum = int(iteration['num'])
if self.params['package'] != 'EMAN':
postrefine = True
else:
postrefine = False
apEulerDraw.createEulerImages(refrunid,
iternum,
path=self.params['rundir'],
postrefine=postrefine)
return
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
# symmetry info
if self.params['sym'] == 'Icos':
self.params['symtype'] = 'I'
self.params['symfold'] = None
else:
self.params['symtype'] = self.params['sym'][0]
self.params['symfold'] = int(self.params['sym'][1:])
# eman "d" symmetry is spider "ci"
if self.params['symtype'].upper() == "D":
self.params['symtype'] = "CI"
# create symmetry doc file
sydoc = "sym.spi"
refine.symmetryDoc(self.params['symtype'], self.params['symfold'],
sydoc)
# convert incr to array of increments
ang_inc = self.params['incr'].split(',')
self.params['numiter'] = len(ang_inc)
self.params['increments'] = []
for i in range(0, self.params['numiter']):
self.params['increments'].append(int(ang_inc[i]))
# convert stack to spider stack
spiderstack = os.path.join(self.params['rundir'], 'start.spi')
operations.stackToSpiderStack(self.stack['file'],
spiderstack,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
numpart=self.params['numpart'])
# create filtered stack
spiderstackfilt = spiderstack
if (self.params['lowpass'] + self.params['highpass']) > 0:
spiderstackfilt = os.path.join(self.params['rundir'],
'start_filt.spi')
operations.stackToSpiderStack(self.stack['file'],
spiderstackfilt,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
lp=self.params['lowpass'],
hp=self.params['highpass'],
numpart=self.params['numpart'])
# rescale initial model if necessary
outvol = os.path.join(self.params['rundir'], "vol000.spi")
apModel.rescaleModel(self.params['modelid'],
outvol,
self.stack['boxsize'],
self.stack['apix'],
spider=True)
self.params['itervol'] = outvol
for iter in range(1, self.params['numiter'] + 1):
# create projections for projection matching
apDisplay.printMsg("creating reference projections of volume: %s" %
self.params['itervol'])
projs, numprojs, ang, sel = refine.createProjections(
incr=self.params['increments'][iter - 1],
boxsz=self.stack['boxsize'],
symfold=self.params['symfold'],
invol=self.params['itervol'],
rad=self.params['rad'],
)
# run reference-based alignment
apDisplay.printMsg("running reference-based alignment (AP MQ)")
apmqfile = "apmq%03d.spi" % iter
outang = "angular%03d.spi" % iter
shf = "shifts%03d.spi" % iter
shiftedStack = "parts_shifted.spi"
apFile.removeFile(shf)
refine.spiderAPMQ(
projs=projs,
numprojs=numprojs,
tsearch=self.params['xysearch'],
tstep=self.params['xystep'],
firstRing=self.params['firstring'],
lastRing=self.params['lastring'],
stackfile=spiderstackfilt,
nump=self.params['numpart'],
ang=ang,
apmqfile=apmqfile,
outang=outang,
nproc=self.params['proc'],
)
# use cross-correlation to find the sub-pixel alignment
# of the particles,
# results will be saved in "peakfile.spi"
apFile.removeFile(shiftedStack)
# don't use MPI here - for some reason slower?
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
apmqlist = refine.readDocFile(apmqfile)
avgccrot = 0
apDisplay.printMsg("creating shifted stack")
for p in range(0, self.params['numpart']):
ref = int(float(apmqlist[p][2]))
ccrot = float(apmqlist[p][3])
inplane = float(apmqlist[p][4])
avgccrot += ccrot
# invert the sign - ref projs will be rotated
inplane *= -1
# get corresponding projection
if (ref <= 0):
# mirror projection if necessary
ref *= -1
refimg = spyder.fileFilter(projs) + "@" + str(ref)
refine.mirrorImg(refimg, "_3", inMySpi=mySpi)
img = "_3"
else:
img = spyder.fileFilter(projs) + "@" + str(ref)
refine.rotAndShiftImg(img, "_2", inplane, inMySpi=mySpi)
refine.maskImg("_2",
"_3",
self.params['rad'],
"D",
"E",
center=int((self.stack['boxsize'] / 2) + 1),
inMySpi=mySpi)
# pad ref image & stack image to twice the size
refine.padImg("_3",
"_2",
2 * self.stack['boxsize'],
"N",
1,
1,
0,
inMySpi=mySpi)
stackimg = spyder.fileFilter(spiderstack) + "@" + str(p + 1)
refine.padImg(stackimg,
"_1",
2 * self.stack['boxsize'],
"B",
1,
1,
inMySpi=mySpi)
# calculate cross-correlation
refine.getCC("_1", "_2", "_1", inMySpi=mySpi)
# crop the correllation image to allowable shift amount
shift = int(self.params['allowedShift'] *
self.stack['boxsize'])
dim = 2 * shift + 1
topleftx = self.stack['boxsize'] - shift + 1
refine.windowImg("_1",
"_2",
dim,
topleftx,
topleftx,
inMySpi=mySpi)
# find the sub-pixel location of cc peak
mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17", "_2",
"0")
# create new stack of shifted particles
shpos = spyder.fileFilter(shiftedStack) + "@" + str(p + 1)
mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN")
mySpi.toSpiderQuiet("GP x17", "_2",
str(shift + 1) + "," + str(shift + 1))
refine.copyImg(stackimg, shpos, inMySpi=mySpi)
mySpi.toSpiderQuiet("ELSE")
#mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16")
mySpi.toSpiderQuiet("SH F", stackimg, shpos, "-x15,-x16")
mySpi.toSpiderQuiet("ENDIF")
# save shifts to file
mySpi.toSpiderQuiet("SD " + str(p + 1) + ",x15,x16,x17",
spyder.fileFilter(shf))
mySpi.toSpiderQuiet("SD E", spyder.fileFilter(shf))
mySpi.close()
# create class average images
refine.createClassAverages(
shiftedStack,
projs,
apmqfile,
numprojs,
self.stack['boxsize'],
shifted=True,
)
# rename class averages & variacnes for iteration
cmd = "/bin/mv classes.hed classes.%d.hed;" % iter
cmd += "/bin/mv classes.img classes.%d.img;" % iter
cmd += "/bin/mv variances.hed variances.%d.hed;" % iter
cmd += "/bin/mv variances.img variances.%d.img;" % iter
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
# calculate the stddev for the apmq cc's for throwing out particles
avgccrot /= self.params['numpart']
stdccrot = 0
for p in range(0, self.params['numpart']):
stdccrot += abs(float(apmqlist[p][3]) - avgccrot)
stdccrot /= self.params['numpart']
cccutoff = avgccrot + (stdccrot * self.params['keepsig'])
apDisplay.printMsg("average cc: %f" % avgccrot)
apDisplay.printMsg("setting cutoff to: %f" % cccutoff)
# create new selection file that only has particles with good cc's
selectfile = "select%03d.spi" % iter
apFile.removeFile(selectfile)
mySpi = spyder.SpiderSession(nproc=self.params['proc'],
dataext=".spi",
logo=False,
log=False)
i = 1
for p in range(0, self.params['numpart']):
ccrot = float(apmqlist[p][3])
if ccrot >= cccutoff:
mySpi.toSpiderQuiet("x11=%d" % (p + 1))
mySpi.toSpiderQuiet("SD %d,x11" % i,
spyder.fileFilter(selectfile))
i += 1
mySpi.close()
# calculate the new 3d structure using centered projections
# and the corrected angles from the angular doc file
apDisplay.printMsg("creating 3d volume")
out_rawvol = "vol_raw%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
selectfile,
rad=self.params['rad'],
ang=outang,
out=out_rawvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['bpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'],
)
else:
refine.backProjection(shiftedStack,
selectfile,
ang=outang,
out=out_rawvol,
sym=sydoc,
nproc=self.params['proc'])
# create even & odd select files
apDisplay.printMsg("creating even/odd volumes")
oddfile = "selectodd%03d.spi" % iter
evenfile = "selecteven%03d.spi" % iter
refine.docSplit(selectfile, oddfile, evenfile)
# get the even & odd volumesa
oddvol = "vol1%03d.spi" % iter
evenvol = "vol2%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
oddfile,
rad=self.params['rad'],
ang=outang,
out=oddvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'])
refine.iterativeBackProjection(
shiftedStack,
evenfile,
rad=self.params['rad'],
ang=outang,
out=evenvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'])
else:
refine.backProjection(
shiftedStack,
oddfile,
ang=outang,
out=oddvol,
sym=sydoc,
nproc=self.params['proc'],
)
refine.backProjection(
shiftedStack,
evenfile,
ang=outang,
out=evenvol,
sym=sydoc,
nproc=self.params['proc'],
)
# calculate the FSC
apDisplay.printMsg("calculating FSC")
fscfile = "fsc%03d.spi" % iter
emanfsc = "fsc.eotest.%d" % iter
refine.calcFSC(oddvol, evenvol, fscfile)
# convert to eman-style fscfile
refine.spiderFSCtoEMAN(fscfile, emanfsc)
# calculate the resolution at 0.5 FSC & write to file
res = apRecon.calcRes(emanfsc, self.stack['boxsize'],
self.stack['apix'])
restxt = "resolution.txt"
if iter == 1 and os.path.isfile(restxt):
os.remove(restxt)
resfile = open(restxt, "a")
resfile.write("iter %d:\t%.3f\n" % (iter, res))
resfile.close()
# filter & normalize the volume to be used as a reference in the next round
outvol = "vol%03d.spi" % iter
emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % (
out_rawvol, outvol, self.stack['apix'], res,
self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# create mrc files of volumes
emancmd = "proc3d %s %s apix=%.3f mask=%d norm" % (
out_rawvol, "threed.%da.mrc" % iter, self.stack['apix'],
self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % (
out_rawvol, "threed.%da.lp.mrc" % iter, self.stack['apix'],
res, self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.param['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# set this model as start for next iteration, remove previous
os.remove(self.params['itervol'])
os.remove(out_rawvol)
self.params['itervol'] = outvol
# clean up directory
apDisplay.printMsg("cleaning up directory")
if os.path.isfile(oddvol):
os.remove(oddvol)
if os.path.isfile(evenvol):
os.remove(evenvol)
if os.path.isfile(ang):
os.remove(ang)
if os.path.isfile(sel):
os.remove(sel)
if os.path.isfile(projs):
os.remove(projs)
if os.path.isfile(oddfile):
os.remove(oddfile)
if os.path.isfile(evenfile):
os.remove(evenfile)
if os.path.isfile(emanfsc) and os.path.isfile(fscfile):
os.remove(fscfile)
if os.path.isfile(shiftedStack):
os.remove(shiftedStack)
os.remove(self.params['itervol'])
def insertIteration(self, iteration):
refineparamsq=appiondata.ApEmanRefineIterData()
refineparamsq['ang']=iteration['ang']
refineparamsq['lpfilter']=iteration['lpfilter']
refineparamsq['hpfilter']=iteration['hpfilter']
refineparamsq['pad']=iteration['pad']
refineparamsq['EMAN_maxshift']=iteration['maxshift']
refineparamsq['EMAN_hard']=iteration['hard']
refineparamsq['EMAN_classkeep']=iteration['classkeep']
refineparamsq['EMAN_classiter']=iteration['classiter']
refineparamsq['EMAN_filt3d']=iteration['filt3d']
refineparamsq['EMAN_shrink']=iteration['shrink']
refineparamsq['EMAN_euler2']=iteration['euler2']
refineparamsq['EMAN_xfiles']=iteration['xfiles']
refineparamsq['EMAN_median']=iteration['median']
refineparamsq['EMAN_phasecls']=iteration['phasecls']
refineparamsq['EMAN_fscls']=iteration['fscls']
refineparamsq['EMAN_refine']=iteration['refine']
refineparamsq['EMAN_goodbad']=iteration['goodbad']
refineparamsq['EMAN_perturb']=iteration['perturb']
refineparamsq['MsgP_cckeep']=iteration['msgpasskeep']
refineparamsq['MsgP_minptls']=iteration['msgpassminp']
#create Chimera snapshots
fscfile = os.path.join(self.params['rundir'], "fsc.eotest."+iteration['num'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix'])
if self.params['snapfilter']:
halfres = self.params['snapfilter']
volumeDensity = 'threed.'+iteration['num']+'a.mrc'
volDensPath = os.path.join(self.params['rundir'], volumeDensity)
apChimera.filterAndChimera(volDensPath, halfres, self.params['apix'],
self.params['boxsize'], 'snapshot', self.params['contour'], self.params['zoom'],
sym=iteration['sym']['eman_name'], mass=self.params['mass'])
## uncommment this for chimera image only runs...
if self.params['chimeraonly'] is True:
return
# insert resolution data
if halfres != True:
resData = self.getResolutionData(iteration)
else:
apDisplay.printWarning("resolution reported as nan, not committing results to database")
return
if self.params['package']== 'EMAN':
refineclassavg='classes_eman.'+iteration['num']+'.img'
postrefineclassavg=None
elif self.params['package']== 'EMAN/SpiCoran':
refineclassavg='classes_eman.'+iteration['num']+'.img'
postrefineclassavg='classes_coran.'+iteration['num']+'.img'
elif self.params['package']== 'EMAN/MsgP':
refineclassavg='classes_eman.'+iteration['num']+'.img'
postrefineclassavg='classes_msgp.'+iteration['num']+'.img'
else:
apDisplay.printError("Refinement Package Not Valid")
# insert refinement results
refineq = appiondata.ApRefineIterData()
refineq['refineRun'] = self.params['refineRun']
refineq['emanParams'] = refineparamsq
refineq['iteration'] = iteration['num']
refineq['resolution'] = resData
refineq['rMeasure'] = self.getRMeasureData(iteration)
refineq['mask'] = iteration['mask']
refineq['imask'] = iteration['imask']
refineq['symmetry']=iteration['sym']
refineq['exemplar'] = False
classvar = 'classes.'+iteration['num']+'.var.img'
refineq['refineClassAverages'] = refineclassavg
refineq['postRefineClassAverages'] = postrefineclassavg
if classvar in self.params['classvars']:
refineq['classVariance'] = classvar
if volumeDensity in self.params['volumes']:
refineq['volumeDensity'] = volumeDensity
apDisplay.printMsg("inserting Refinement Data into database")
if self.params['commit'] is True:
refineq.insert()
else:
apDisplay.printWarning("not committing results to database")
#insert FSC data
fscfile = os.path.join(self.params['rundir'], "fsc.eotest."+iteration['num'])
self.insertFSC(fscfile, refineq, self.params['commit'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix'])
apDisplay.printColor("FSC 0.5 Resolution: "+str(halfres), "cyan")
# get projections eulers for iteration:
eulers = self.getEulersFromProj(iteration['num'])
# get list of bad particles for this iteration
badprtls = self.readParticleLog(self.params['rundir'], iteration['num'])
# expand cls.*.tar into temp file
clsf=os.path.join(self.params['rundir'], "cls."+iteration['num']+".tar")
#print "reading",clsf
clstar=tarfile.open(clsf)
clslist=clstar.getmembers()
clsnames=clstar.getnames()
#print "extracting",clsf,"into temp directory"
for clsfile in clslist:
clstar.extract(clsfile,self.params['tmpdir'])
clstar.close()
# for each class, insert particle alignment info into database
apDisplay.printColor("Inserting Particle Classification Data for "
+str(len(clsnames))+" classes", "magenta")
t0 = time.time()
for cls in clsnames:
self.insertRefineParticleData(cls, iteration, eulers, badprtls, refineq, len(clsnames))
apDisplay.printColor("\nFinished in "+apDisplay.timeString(time.time()-t0), "magenta")
# remove temp directory
for file in os.listdir(self.params['tmpdir']):
os.remove(os.path.join(self.params['tmpdir'],file))
os.rmdir(self.params['tmpdir'])
#create euler freq map
apDisplay.printMsg("creating euler frequency map")
refrunid = int(self.params['refineRun'].dbid)
iternum = int(iteration['num'])
if self.params['package'] != 'EMAN':
postrefine = True
else:
postrefine = False
apEulerDraw.createEulerImages(refrunid, iternum, path=self.params['rundir'], postrefine=postrefine)
return
