def processFrealignVolume(self, iternum, symname):
volumeImagicFile = os.path.join(self.params['rundir'],
"threed.%03da.hed" % (iternum))
if not os.path.isfile(volumeImagicFile):
apDisplay.printError("Failed to find volume: %s" %
(volumeImagicFile))
volumeMrcFile = os.path.join(self.params['rundir'],
"threed.%03da.mrc" % (iternum))
if not os.path.isfile(volumeMrcFile):
emancmd = "proc3d %s %s apix=%.3f origin=0,0,0 norm=0,1" % (
volumeImagicFile, volumeMrcFile, self.apix)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
if not os.path.isfile(volumeMrcFile):
apDisplay.printError("Failed to convert volume")
apChimera.filterAndChimera(volumeMrcFile,
res=10,
apix=self.apix,
chimtype='snapshot',
contour=1.0,
zoom=self.params['zoom'],
sym=symname,
silhouette=True,
mass=self.params['mass'])
return volumeMrcFile
def createChimeraVolumeSnapshot(self,
volume,
iteration,
reference_number=1):
''' create chimera snapshot images of volume '''
if self.params['snapfilter']:
resolution = self.params['snapfilter']
else:
newfscfile = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.fsc" \
% (self.params['timestamp'], iteration, reference_number))
try:
resolution = apRecon.getResolutionFromGenericFSCFile(
newfscfile, self.runparams['boxsize'],
self.runparams['apix'])
except:
apDisplay.printWarning(
"Failed to get resolution from generic FSC file: " +
newfscfile)
resolution = 30
apDisplay.printWarning(
"Running Chimera Snapshot with resolution: %d " % resolution)
apChimera.filterAndChimera(volume,
resolution,
self.runparams['apix'],
self.runparams['boxsize'],
'snapshot',
self.params['contour'],
self.params['zoom'],
sym='c1',
mass=self.params['mass'])
def createChimeraVolumeSnapshot(self, volume, iteration, reference_number=1):
""" create chimera snapshot images of volume """
if self.params["snapfilter"]:
resolution = self.params["snapfilter"]
else:
newfscfile = os.path.join(
self.resultspath,
"recon_%s_it%.3d_vol%.3d.fsc" % (self.params["timestamp"], iteration, reference_number),
)
try:
resolution = apRecon.getResolutionFromGenericFSCFile(
newfscfile, self.runparams["boxsize"], self.runparams["apix"]
)
except:
apDisplay.printWarning("Failed to get resolution from generic FSC file: " + newfscfile)
resolution = 30
apDisplay.printWarning("Running Chimera Snapshot with resolution: %d " % resolution)
apChimera.filterAndChimera(
volume,
resolution,
self.runparams["apix"],
self.runparams["boxsize"],
"snapshot",
self.params["contour"],
self.params["zoom"],
sym="c1",
mass=self.params["mass"],
)
def insertHipIterData(self):
for i in range(3):
iternum = i + 1
apDisplay.printColor("\nUploading iteration %d of %d\n" % (iternum, 3), "green")
HipIter = appiondata.ApHipIterData()
sessiondata = apDatabase.getSessionDataFromSessionName(self.params["sessionname"])
HipIter["session"] = sessiondata
if iternum == 1:
avgpath = os.path.join(self.params["rundir"], "avgsnif1")
HipIter["chop1"] = os.path.join(avgpath, "chop1.dek")
elif iternum == 2:
avgpath = os.path.join(self.params["rundir"], "avgsnif1/avgsnif2")
HipIter["chop2"] = os.path.join(avgpath, "chop2.dek")
elif iternum == 3:
avgpath = os.path.join(self.params["rundir"], "avgsnif1/avgsnif2/avg3")
volumeMrcFile = os.path.join(avgpath, "avglist3_%dp.mrc" % (self.params["rescut"]))
apChimera.filterAndChimera(
volumeMrcFile,
res=10,
apix=self.params["step"],
chimtype="snapshot",
contour=1.0,
zoom=1,
sym="c9",
silhouette=True,
)
HipIter["hipRun"] = self.params["HipRun"]
HipIter["iteration"] = iternum
HipIter["iterpath"] = avgpath
HipIter["volumeDensity"] = volumeMrcFile
HipIter["resolution"] = self.getResolutionData(avgpath, iternum)
HipIter["rMeasure"] = self.getRMeasureData(volumeMrcFile)
HipIter["cutfit1"] = os.path.join(avgpath, "cutfit1.dek")
HipIter["cutfit2"] = os.path.join(avgpath, "cutfit2.dek")
HipIter["cutfit3"] = os.path.join(avgpath, "cutfit3.dek")
HipIter["avglist_file"] = os.path.join(avgpath, "avglist3_%dp.list" % (self.params["rescut"]))
f = open(HipIter["avglist_file"], "r")
lines = f.readlines()
f.close()
HipIter["final_numpart"] = len(lines)
HipIter["asymsu"] = int(
((self.params["boxsize"] * self.params["step"]) / self.params["replen"])
* self.params["subunits"]
* len(lines)
)
HipIter["avg_file"] = os.path.join(avgpath, "avglist3_%dp.avg" % (self.params["rescut"]))
HipIter["map_file"] = os.path.join(avgpath, "avglist3_%dp.map" % (self.params["rescut"]))
HipIter["mrc_file"] = os.path.join(avgpath, "avglist3_%dp.mrc" % (self.params["rescut"]))
HipIter["ll_file"] = os.path.join(avgpath, "pll.list")
HipIter["op_file"] = os.path.join(avgpath, "pop.list")
HipIter["output_file"] = os.path.join(avgpath, "avglist3_%dp.OUTPUT" % (self.params["rescut"]))
if self.params["commit"] is True:
HipIter.insert()
else:
print HipIter
self.params["HipIterID"] = HipIter.dbid
print "self.params['HipIterID']", self.params["HipIterID"]
return
def processFrealignVolume(self, iternum, symname):
volumeImagicFile = os.path.join(self.params['rundir'], "threed.%03da.hed"%(iternum))
if not os.path.isfile(volumeImagicFile):
apDisplay.printError("Failed to find volume: %s"%(volumeImagicFile))
volumeMrcFile = os.path.join(self.params['rundir'], "threed.%03da.mrc"%(iternum))
if not os.path.isfile(volumeMrcFile):
emancmd = "proc3d %s %s apix=%.3f origin=0,0,0 norm=0,1"%(volumeImagicFile, volumeMrcFile, self.apix)
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
if not os.path.isfile(volumeMrcFile):
apDisplay.printError("Failed to convert volume")
apChimera.filterAndChimera(volumeMrcFile, res=10, apix=self.apix, chimtype='snapshot',
contour=1.0, zoom=self.params['zoom'], sym=symname, silhouette=True, mass=self.params['mass'])
return volumeMrcFile
def insertHipIterData(self):
for i in range(3):
iternum = i+1
apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, 3), "green")
HipIter = appiondata.ApHipIterData()
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
HipIter['session'] = sessiondata
if iternum == 1:
avgpath = os.path.join(self.params['rundir'], "avgsnif1")
HipIter['chop1'] = os.path.join(avgpath, "chop1.dek")
elif iternum == 2:
avgpath = os.path.join(self.params['rundir'], "avgsnif1/avgsnif2")
HipIter['chop2'] = os.path.join(avgpath, "chop2.dek")
elif iternum == 3:
avgpath = os.path.join(self.params['rundir'], "avgsnif1/avgsnif2/avg3")
volumeMrcFile = os.path.join(avgpath, "avglist3_%dp.mrc"%(self.params['rescut']))
apChimera.filterAndChimera(volumeMrcFile, res=10, apix=self.params['step'], chimtype='snapshot', contour=1.0, zoom=1, sym='c9', silhouette=True)
HipIter['hipRun'] = self.params['HipRun']
HipIter['iteration'] = iternum
HipIter['iterpath'] = avgpath
HipIter['volumeDensity'] = volumeMrcFile
HipIter['resolution'] = self.getResolutionData(avgpath, iternum)
HipIter['rMeasure'] = self.getRMeasureData(volumeMrcFile)
HipIter['cutfit1'] = os.path.join(avgpath, "cutfit1.dek")
HipIter['cutfit2'] = os.path.join(avgpath, "cutfit2.dek")
HipIter['cutfit3'] = os.path.join(avgpath, "cutfit3.dek")
HipIter['avglist_file'] = os.path.join(avgpath, "avglist3_%dp.list"%(self.params['rescut']))
f = open(HipIter['avglist_file'], 'r')
lines = f.readlines()
f.close()
HipIter['final_numpart'] = len(lines)
HipIter['asymsu'] = int(((self.params['boxsize']*self.params['step'])/self.params['replen'])*self.params['subunits']*len(lines))
HipIter['avg_file'] = os.path.join(avgpath, "avglist3_%dp.avg"%(self.params['rescut']))
HipIter['map_file'] = os.path.join(avgpath, "avglist3_%dp.map"%(self.params['rescut']))
HipIter['mrc_file'] = os.path.join(avgpath, "avglist3_%dp.mrc"%(self.params['rescut']))
HipIter['ll_file'] = os.path.join(avgpath, "pll.list")
HipIter['op_file'] = os.path.join(avgpath, "pop.list")
HipIter['output_file'] = os.path.join(avgpath, "avglist3_%dp.OUTPUT"%(self.params['rescut']))
if self.params['commit'] is True:
HipIter.insert()
else:
print HipIter
self.params['HipIterID'] = HipIter.dbid
print "self.params['HipIterID']",self.params['HipIterID']
return
def start(self):
'''
OptiMod script
'''
############### define short-hand parameter names ############
refine = True
rundir = self.params['rundir']
box = self.params['boxsize']
rbox = self.params['refineboxsize']
apix = self.params['apix']
rapix = self.params['refineapix']
npart = self.params['numpart']
nvol = self.params['num_volumes']
# sym = self.params['sym']
sym = "c1"
psym = self.params['presumed_sym']
first = self.params['firstimage']
nws = self.params['non_weighted_sequence']
threes = self.params['threes']
lmask = self.params['linmask']
asqfilt = self.params['asqfilt']
anginc = self.params['ang_inc']
keep_ordered = self.params['keep_ordered_num']
hamwin = self.params['ham_win']
useEMAN1 = self.params['useEMAN1']
ipv = self.params['images_per_volume']
lp = self.params['3d_lpfilt']
nref = self.params['nref']
PCA = self.params['PCA']
neigens = self.params['numeigens']
recalc = self.params['recalculate']
preftype = self.params['preftype']
mrad = self.params['mask_radius']
irad = self.params['inner_radius']
orad = self.params['outer_radius']
tnproc = self.params['threadnproc']
nproc = self.params['nproc']
oldavgs = self.params['oldavgs']
avgs = os.path.join(self.params['rundir'], self.params['avgs'])
start = self.params['start_at']
############# copy to working directory ############
if not os.path.isfile(os.path.join(avgs)[:-4]+".hed"):
shutil.copyfile(os.path.join(oldavgs[:-4]+".hed"), avgs[:-4]+".hed")
if not os.path.isfile(os.path.join(avgs)[:-4]+".img"):
shutil.copyfile(os.path.join(oldavgs[:-4]+".img"), avgs[:-4]+".img")
if self.params['ravgs'] is not None:
ravgs = os.path.join(self.params['rundir'], self.params['ravgs'])
if not os.path.isfile(self.params['ravgs'][:-4]+".hed"):
shutil.copy(self.params['oldravgs'][:-4]+".hed",
os.path.join(self.params['ravgs'][:-4]+".hed"))
if not os.path.isfile(self.params['ravgs'][:-4]+".img"):
shutil.copy(self.params['oldravgs'][:-4]+".img",
os.path.join(self.params['ravgs'][:-4]+".img"))
### Euler jumper assessment does not make sense when refine images are different
ejassess = False
else:
ravgs = avgs
ejassess = True
########### scale & prealign class averages, if specified #########
if self.params['scale'] is True:
emancmd = "proc2d %s %s_scaled.img scale=%.3f clip=%i,%i edgenorm" \
% (avgs, avgs[:-4], self.scalefactor, 64, 64)
avgs = avgs[:-4]+"_scaled.img"
if start == "none":
while os.path.isfile(avgs):
apFile.removeStack(avgs)
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printColor("skipping stack scaling", "cyan")
if self.imagicroot is not None and useEMAN1 is False:
apIMAGIC.takeoverHeaders(avgs, npart, box)
if self.imagicroot is not None:
if self.params['prealign'] is True:
if start == "none":
avgs = apIMAGIC.prealignClassAverages(rundir, avgs)
apIMAGIC.checkLogFileForErrors(
os.path.join(rundir, "prealignClassAverages.log"))
### select between EMAN1 and IMAGIC for common lines
if self.imagicroot is not None and useEMAN1 is False:
commonlinesdir = os.path.join(rundir, "angular_reconstitution")
else:
commonlinesdir = os.path.join(rundir, "cross_common_lines")
### make links & calculate CCC matrix for 2D averages
apDisplay.printColor("Calculating similarity matrix", "cyan")
ccc_matrix = apCommonLines.calculate_ccc_matrix_2d(avgs)
clsavgs = os.path.split(avgs)[1][:-4]
if not os.path.isdir(commonlinesdir):
os.mkdir(commonlinesdir)
if os.path.islink(os.path.join(commonlinesdir, clsavgs+".hed")):
os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".hed"))
os.symlink(os.path.join(rundir, clsavgs+".hed"),
os.path.join(commonlinesdir, clsavgs+".hed"))
if os.path.islink(os.path.join(commonlinesdir, clsavgs+".img")):
os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".img"))
os.symlink(os.path.join(rundir, clsavgs+".img"),
os.path.join(commonlinesdir, clsavgs+".img"))
################## create multiple 3d0s #####################
if start == "none":
if self.imagicroot is not None and useEMAN1 is False:
cmdlist = []
seqfile = open(os.path.join(rundir,
"sequences_for_angular_reconstitution.dat"), "w")
apDisplay.printColor("Running multiple raw volume calculations", "cyan")
for i in range(nvol):
sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart,
ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
apCommonLines.check_for_duplicates_in_sequence(sequence)
seqfile.write(str(sequence)+"\n")
### create IMAGIC batch file for each model & append them to be threaded
batchfile = apCommonLines.imagic_batch_file(sequence, i+1, avgs, sym,
asqfilt, lmask, apix, box, anginc, keep_ordered, hamwin, lp,
threes=threes, do_not_remove=False)
proc = subprocess.Popen('chmod 755 '+batchfile, shell=True)
proc.wait()
cmdlist.append(batchfile)
os.chdir(rundir)
seqfile.close()
apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)
### check for errors after execution
for i in range(nvol):
apIMAGIC.checkLogFileForErrors(
os.path.join(commonlinesdir, "3d"+str(i+1)+".log"))
else:
### use EMAN1 cross-common lines
cmdlist = []
seqfile = open(os.path.join(rundir, "sequences_for_cross_common_lines.dat"), "w")
for i in range(nvol):
sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart,
ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
apCommonLines.check_for_duplicates_in_sequence(sequence)
seqfile.write(str(sequence)+"\n")
vol_avgfile = os.path.join(commonlinesdir, "vol_%d_averages.hed" % (i+1))
vol_seqfile = os.path.join(commonlinesdir, "vol_%d_averages.txt" % (i+1))
vsf = open(vol_seqfile, "w")
for j in range(ipv):
vsf.write("%d\n" % sequence[j])
vsf.close()
tmpdir = os.path.join(commonlinesdir, "tmp%d" % (i+1))
fullcmd = "proc2d %s %s list=%s ; " % (avgs, vol_avgfile, vol_seqfile)
fullcmd+= "rm -rf %s ; mkdir %s ; cd %s ; " % (tmpdir, tmpdir, tmpdir)
fullcmd+= "startAny %s sym=c1 proc=1 rounds=2 mask=%d ; " \
% (vol_avgfile, mrad/apix)
fullcmd+= "mv threed.0a.mrc ../threed_%d.mrc ; " % (i+1)
fullcmd+= "mv CCL.hed ../CCL_%d.hed ; " % (i+1)
fullcmd+= "mv CCL.img ../CCL_%d.img ; " % (i+1)
cmdlist.append(fullcmd)
seqfile.close()
apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)
for i in range(nvol):
shutil.rmtree(os.path.join(commonlinesdir, "tmp%d" % (i+1)))
else:
apDisplay.printColor("skipping common lines volume calculations", "cyan")
############### move 3-D models volume directory ################
### create volume directory
volumedir = os.path.join(rundir, "volumes")
if not os.path.isdir(volumedir):
os.mkdir(volumedir)
volumes = {}
cmds = []
if start == "none":
apDisplay.printColor("moving volumes for Xmipp 3-D Maximum Likelihood", "cyan")
for i in range(nvol):
if useEMAN1 is False:
volume1 = os.path.join(commonlinesdir, "3d%d_ordered%d_filt.vol" % (i+1,i+1))
else:
volume1 = os.path.join(commonlinesdir, "threed_%d.mrc" % (i+1))
volume2 = os.path.join(volumedir, "3d%d.vol" % (i+1))
if start == "none":
if useEMAN1 is False:
shutil.move(volume1, volume2)
else:
cmds.append("proc3d %s %s spidersingle" % (volume1, volume2))
volumes[(i+1)] = volume2
apThread.threadCommands(cmds, nproc=tnproc)
#################### align 3-D models ##################
if start == "3D_align" or start == "none":
# if start != "3D_assess" and start != "3D_refine":
### run Maximum Likelihood 3-D alignment & align resulting volumes
apDisplay.printColor("Running Xmipp maximum likelihood 3-D alignment", "cyan")
vol_doc_file, alignref = apCommonLines.xmipp_max_like_3d_align(
volumes, nproc, tnproc, nref)
alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)
apDisplay.printColor("Aligning volumes based on 3-D ML parameters", "cyan")
apCommonLines.align_volumes(alignparams, alignref, nvol, tnproc, apix)
else:
apDisplay.printColor("skipping 3D alignment", "cyan")
vol_doc_file, alignref = apCommonLines.findAlignmentParams(
os.path.join(rundir,"max_like_alignment"), nref)
alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)
##################### Principal Component Analysis ##############
apDisplay.printColor("Calculating inter-volume similarity", "cyan")
aligned_volumes = {}
for i in range(nvol):
aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.vol" % (i+1))
# aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.mrc" % (i+1))
if PCA is True:
simfile, sim_matrix = apCommonLines.runPrincipalComponentAnalysis(
aligned_volumes, nvol, neigens, box, apix, recalculate=recalc)
else:
simfile, sim_matrix = apCommonLines.calculate_ccc_matrix_3d(
aligned_volumes, nvol)
################ 3-D affinity propagation #############
### 3-D Affinity Propagation
apDisplay.printColor("Averaging volumes with Affinity Propagation", "cyan")
preffile = apCommonLines.set_preferences(sim_matrix, preftype, nvol)
classes = apCommonLines.run_affinity_propagation(aligned_volumes, simfile,
preffile, box, apix)
######### refine volumes using Xmipp projection matching #########
if start == "3D_align" or start == "3D_refine" or start == "none":
# if start != "3D_assess":
if not os.path.isdir("refinement"):
os.mkdir("refinement")
os.chdir("refinement")
apXmipp.breakupStackIntoSingleFiles(ravgs)
xmippcmd = "xmipp_normalize -i partlist.sel -method OldXmipp"
apParam.runCmd(xmippcmd, "Xmipp")
for i in classes.keys():
emancmd = "proc3d %s %s scale=%.3f clip=%d,%d,%d mask=%s spidersingle" \
% (os.path.join(rundir, "%d.mrc" % i), "%d.vol" % i, \
(1/self.scalefactor), rbox, rbox, rbox, (mrad / rapix))
apParam.runCmd(emancmd, "EMAN")
apCommonLines.refine_volume(rundir, ravgs, i, mrad, irad, orad, rapix, nproc)
emancmd = "proc3d %s %s apix=%.3f" \
% (os.path.join("refine_%d" % i, "3d%d_refined.vol" % i), \
os.path.join(rundir, "%d_r.mrc" % i), rapix)
apParam.runCmd(emancmd, "EMAN")
os.chdir(rundir)
else:
apDisplay.printColor("skipping 3D refinement", "cyan")
################# model evaluation ################
if start == "3D_align" or start == "3D_refine" or start == "3D_assess" or start == "none":
# if start == "none" or start=="3D_assess":
### final model assessment
try:
euler_array = apCommonLines.getEulerValuesForModels(alignparams,
rundir, nvol, npart, threes=threes)
except:
euler_array = None
ejassess = False
if refine is True:
apCommonLines.assess_3Dclass_quality2(rundir, aligned_volumes, sim_matrix,
classes, euler_array, box, rbox, rapix, ravgs, psym, npart, ejassess=ejassess)
else:
apCommonLines.assess_3Dclass_quality(rundir, aligned_volumes, sim_matrix,
classes, euler_array, box, apix, avgs, psym, npart, ejassess=ejassess)
apCommonLines.combineMetrics("final_model_stats.dat",
"final_model_stats_sorted_by_Rcrit.dat", **{"CCPR":(1,1)})
################## upload & cleanup ##################
### upload to database, if specified
self.upload()
### make chimera snapshots
if refine is True:
for i in classes.keys():
if self.params['mass'] is not None:
apChimera.filterAndChimera(
os.path.join(self.params['rundir'], "%d_r.mrc" % i),
res=self.params['3d_lpfilt'],
apix=self.params['refineapix'],
box=self.params['refineboxsize'],
chimtype="snapshot",
contour=2,
zoom=1,
sym="c1",
color="gold",
mass=self.params['mass']
)
### cleanup
snapshots = glob.glob("*.png")
mtlfiles = glob.glob("*.mtl")
objfiles = glob.glob("*.obj")
pyfiles = glob.glob("*mrc.py")
for file in mtlfiles:
os.remove(file)
for file in objfiles:
os.remove(file)
for file in pyfiles:
os.remove(file)
if not os.path.isdir("snapshots"):
os.mkdir("snapshots")
for s in snapshots:
shutil.move(s, os.path.join("snapshots", s))
def start(self):
### start the outfile name
fileroot = os.path.splitext(self.params['filename'])[0]
fileroot += "-"+self.timestamp
filename = self.params['file'].split('/')[-1]
filepath = self.params['file'].strip(filename)
self.params['box'] = apVolume.getModelDimensions(self.params['file'])
if self.params['ampfile'] is not None:
### run amplitude correction
spifile = apVolume.MRCtoSPI(self.params['file'], self.params['rundir'])
tmpfile = apAmpCorrect.createAmpcorBatchFile(spifile, self.params)
apAmpCorrect.runAmpcor()
### check if spider was successful
fileroot += ".amp"
if not os.path.isfile(tmpfile) :
apDisplay.printError("amplitude correction failed")
### convert amplitude corrected file back to mrc
fileroot += ".amp"
a = spider.read(tmpfile)
outfile = os.path.join( self.params['rundir'], "ampl-fix.mrc" )
mrc.write(a, outfile)
curfile = outfile
elif self.params['bfactor'] is True:
outfile = os.path.join( self.params['rundir'], "bfactor-fix.mrc" )
outfile = apAmpCorrect.applyBfactor(self.params['file'], fscfile=self.params['fscfile'], apix=self.params['apix'], mass=self.params['mass'], outfile=outfile)
curfile = outfile
elif self.params['lrdw'] is True:
lrdwcmd = "s_diffmap2 %s %s %s %s"%(self.params['filepath'],self.params['filename'],self.params['ampx'],self.params['resol'])
proc = subprocess.Popen(lrdwcmd)
print lrdwcmd
else :
### just run proc3d
curfile = self.params['file']
emancmd = "proc3d "+self.params['file']+" "
emancmd = "proc3d "+curfile+" "
if self.params['median'] is not None:
data = mrc.read(curfile)
data = ndimage.median_filter(data, size=self.params['median'])
mrc.write(data, curfile)
emancmd+="apix=%s " %self.params['apix']
if self.params['lp'] is not None:
fileroot += (".lp%d" % ( int(self.params['lp']), ))
emancmd += "lp=%d " %self.params['lp']
if self.params['yflip'] is True:
fileroot += ".yflip"
emancmd +="yflip "
if self.params['invert'] is True:
fileroot += ".inv"
emancmd +="mult=-1 "
if self.params['viper'] is True:
fileroot += ".vip"
emancmd +="icos5fTo2f "
if self.params['mask'] is not None:
# convert ang to pixels
maskpix=int(self.params['mask']/self.params['apix'])
fileroot += (".m%d" % ( int(self.params['mask']), ))
emancmd += "mask=%d " %maskpix
self.params['mask'] = maskpix
if self.params['imask'] is not None:
# convert ang to pixels
maskpix=int(self.params['imask']/self.params['apix'])
fileroot += (".im%d" % ( int(self.params['imask']), ))
emancmd += "imask=%d " %maskpix
self.params['imask'] = maskpix
if self.params['norm'] is True:
fileroot += ".norm"
emancmd += "norm=0,1 "
### add output filename to emancmd string
fileroot += ".mrc"
self.params['name'] = fileroot
outfile = os.path.join(self.params['rundir'], fileroot)
emancmd = re.sub(" apix=",(" %s apix=" % outfile), emancmd)
apEMAN.executeEmanCmd(emancmd)
if self.params['description'] is None:
self.params['description'] = "Volume from recon with amplitude adjustment"
### clean up files created during amp correction
if self.params['ampfile'] is not None:
apFile.removeFile(spifile)
apFile.removeFile(tmpfile)
if self.params['commit'] is True:
symdata = apSymmetry.findSymmetry(self.params['sym'])
print symdata
symmetry = symdata['eman_name']
self.params['reconid'] = self.params['reconiterid']
self.insert3dDensity()
### render chimera images of model
apChimera.filterAndChimera(outfile, res=self.params['res'], apix=self.params['apix'], box=self.params['box'],
chimtype='snapshot', mass=self.params['mass'], contour=self.params['contour'],
zoom=self.params['zoom'], sym=symmetry)
resolution = apRecon.getResolutionFromGenericFSCFile(newfscfile, self.runparams['boxsize'], self.runparams['apix'],filtradius=3, criterion=self.fscResCriterion)
except:
apDisplay.printWarning("Failed to get resolution from generic FSC file: "+newfscfile)
resolution = 30
apDisplay.printWarning("Running Chimera Snapshot with resolution: %d " % resolution)
# TODO: need to work this out.
# symmetry was hard coded to 'c1'. why?
try:
symmetry = self.runparams['symmetry']
except Exception, e:
symmetry = self.runparams['symmetry'].split()[0]
symmetry = apSymmetry.findSymmetry( symmetry )
apChimera.filterAndChimera(volume, resolution, self.runparams['apix'],
self.runparams['boxsize'], 'snapshot', self.params['contour'], self.params['zoom'],
sym=symmetry, mass=self.params['mass'])
#=====================
def calculateEulerJumpsAndGoodBadParticles(self, uploadIterations):
''' calculate euler jumps for entire recon, currently based on EMAN ZXZ convention '''
if self.params['commit'] is True:
reconrunid = self.refinerunq.dbid
### make table entries of good-bad particles
apRecon.setGoodBadParticlesFromReconId(reconrunid)
### verify completed refinements and iterations
refinecomplete = self.verifyNumberOfCompletedRefinements(multiModelRefinementRun=self.multiModelRefinementRun)
def insertIteration(self, iteration):
refineparamsq = appiondata.ApEmanRefineIterData()
refineparamsq['ang'] = iteration['ang']
refineparamsq['lpfilter'] = iteration['lpfilter']
refineparamsq['hpfilter'] = iteration['hpfilter']
refineparamsq['pad'] = iteration['pad']
refineparamsq['EMAN_maxshift'] = iteration['maxshift']
refineparamsq['EMAN_hard'] = iteration['hard']
refineparamsq['EMAN_classkeep'] = iteration['classkeep']
refineparamsq['EMAN_classiter'] = iteration['classiter']
refineparamsq['EMAN_filt3d'] = iteration['filt3d']
refineparamsq['EMAN_shrink'] = iteration['shrink']
refineparamsq['EMAN_euler2'] = iteration['euler2']
refineparamsq['EMAN_xfiles'] = iteration['xfiles']
refineparamsq['EMAN_median'] = iteration['median']
refineparamsq['EMAN_phasecls'] = iteration['phasecls']
refineparamsq['EMAN_fscls'] = iteration['fscls']
refineparamsq['EMAN_refine'] = iteration['refine']
refineparamsq['EMAN_goodbad'] = iteration['goodbad']
refineparamsq['EMAN_perturb'] = iteration['perturb']
refineparamsq['MsgP_cckeep'] = iteration['msgpasskeep']
refineparamsq['MsgP_minptls'] = iteration['msgpassminp']
#create Chimera snapshots
fscfile = os.path.join(self.params['rundir'],
"fsc.eotest." + iteration['num'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'],
self.params['apix'])
if self.params['snapfilter']:
halfres = self.params['snapfilter']
volumeDensity = 'threed.' + iteration['num'] + 'a.mrc'
volDensPath = os.path.join(self.params['rundir'], volumeDensity)
apChimera.filterAndChimera(volDensPath,
halfres,
self.params['apix'],
self.params['boxsize'],
'snapshot',
self.params['contour'],
self.params['zoom'],
sym=iteration['sym']['eman_name'],
mass=self.params['mass'])
## uncommment this for chimera image only runs...
if self.params['chimeraonly'] is True:
return
# insert resolution data
if halfres != True:
resData = self.getResolutionData(iteration)
else:
apDisplay.printWarning(
"resolution reported as nan, not committing results to database"
)
return
if self.params['package'] == 'EMAN':
refineclassavg = 'classes_eman.' + iteration['num'] + '.img'
postrefineclassavg = None
elif self.params['package'] == 'EMAN/SpiCoran':
refineclassavg = 'classes_eman.' + iteration['num'] + '.img'
postrefineclassavg = 'classes_coran.' + iteration['num'] + '.img'
elif self.params['package'] == 'EMAN/MsgP':
refineclassavg = 'classes_eman.' + iteration['num'] + '.img'
postrefineclassavg = 'classes_msgp.' + iteration['num'] + '.img'
else:
apDisplay.printError("Refinement Package Not Valid")
# insert refinement results
refineq = appiondata.ApRefineIterData()
refineq['refineRun'] = self.params['refineRun']
refineq['emanParams'] = refineparamsq
refineq['iteration'] = iteration['num']
refineq['resolution'] = resData
refineq['rMeasure'] = self.getRMeasureData(iteration)
refineq['mask'] = iteration['mask']
refineq['imask'] = iteration['imask']
refineq['symmetry'] = iteration['sym']
refineq['exemplar'] = False
classvar = 'classes.' + iteration['num'] + '.var.img'
refineq['refineClassAverages'] = refineclassavg
refineq['postRefineClassAverages'] = postrefineclassavg
if classvar in self.params['classvars']:
refineq['classVariance'] = classvar
if volumeDensity in self.params['volumes']:
refineq['volumeDensity'] = volumeDensity
apDisplay.printMsg("inserting Refinement Data into database")
if self.params['commit'] is True:
refineq.insert()
else:
apDisplay.printWarning("not committing results to database")
#insert FSC data
fscfile = os.path.join(self.params['rundir'],
"fsc.eotest." + iteration['num'])
self.insertFSC(fscfile, refineq, self.params['commit'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'],
self.params['apix'])
apDisplay.printColor("FSC 0.5 Resolution: " + str(halfres), "cyan")
# get projections eulers for iteration:
eulers = self.getEulersFromProj(iteration['num'])
# get list of bad particles for this iteration
badprtls = self.readParticleLog(self.params['rundir'],
iteration['num'])
# expand cls.*.tar into temp file
clsf = os.path.join(self.params['rundir'],
"cls." + iteration['num'] + ".tar")
#print "reading",clsf
clstar = tarfile.open(clsf)
clslist = clstar.getmembers()
clsnames = clstar.getnames()
#print "extracting",clsf,"into temp directory"
for clsfile in clslist:
clstar.extract(clsfile, self.params['tmpdir'])
clstar.close()
# for each class, insert particle alignment info into database
apDisplay.printColor(
"Inserting Particle Classification Data for " +
str(len(clsnames)) + " classes", "magenta")
t0 = time.time()
for cls in clsnames:
self.insertRefineParticleData(cls, iteration, eulers, badprtls,
refineq, len(clsnames))
apDisplay.printColor(
"\nFinished in " + apDisplay.timeString(time.time() - t0),
"magenta")
# remove temp directory
for file in os.listdir(self.params['tmpdir']):
os.remove(os.path.join(self.params['tmpdir'], file))
os.rmdir(self.params['tmpdir'])
#create euler freq map
apDisplay.printMsg("creating euler frequency map")
refrunid = int(self.params['refineRun'].dbid)
iternum = int(iteration['num'])
if self.params['package'] != 'EMAN':
postrefine = True
else:
postrefine = False
apEulerDraw.createEulerImages(refrunid,
iternum,
path=self.params['rundir'],
postrefine=postrefine)
return
def start(self):
self.params['outputstack'] = os.path.join(self.params['rundir'],
self.params['stackname'])
particles, self.params['refineiter'] = getParticleInfo(
self.params['reconid'], self.params['iter'])
stackdata = particles[0]['particle']['stack']
stack = os.path.join(stackdata['path']['path'], stackdata['name'])
classes, cstats = determineClasses(particles)
rejectlst = []
if self.params['sigma'] is not None:
cutoff = cstats[
'meanquality'] + self.params['sigma'] * cstats['stdquality']
apDisplay.printMsg("Cutoff = " + str(cutoff))
rejectlst = self.removePtclsByQualityFactor(
particles, rejectlst, cutoff)
if self.params['avgjump'] is not None:
rejectlst = self.removePtclsByJumps(particles, rejectlst)
if self.params['rejectlst']:
rejectlst = removePtclsByLst(rejectlst, self.params)
classkeys = classes.keys()
classkeys.sort()
classnum = 0
totalptcls = 0
keepfile = open('keep.lst', 'w')
keepfile.write('#LST\n')
reject = open('reject.lst', 'w')
reject.write('#LST\n')
apDisplay.printMsg("Processing classes")
#loop through classes
for key in classkeys:
# file to hold particles of this class
clsfile = open('clstmp.lst', 'w')
clsfile.write('#LST\n')
classnum += 1
if classnum % 10 == 1:
apDisplay.printMsg(
str(classnum) + " of " + (str(len(classkeys))))
images = EMAN.EMData()
#loop through particles in class
nptcls = 0
for ptcl in classes[key]['particles']:
if ptcl['mirror']:
mirror = 1
else:
mirror = 0
rot = ptcl['euler3']
rot = rot * math.pi / 180
if ptcl['particle']['particleNumber'] not in rejectlst:
l = '%d\t%s\t%f,\t%f,%f,%f,%d\n' % (
ptcl['particle']['particleNumber'] - 1, stack,
ptcl['quality_factor'], rot, ptcl['shiftx'],
ptcl['shifty'], mirror)
keepfile.write(l)
clsfile.write(l)
totalptcls += 1
nptcls += 1
else:
reject.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' %
(ptcl['particle']['particleNumber'] - 1,
stack, ptcl['quality_factor'], rot,
ptcl['shiftx'], ptcl['shifty'], mirror))
#if ptcl['quality_factor']>cstats['meanquality']+3*cstats['stdquality']:
# high.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1,
# stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror))
clsfile.close()
if nptcls < 1:
continue
if self.params['skipavg'] is False:
makeClassAverages('clstmp.lst', self.params['outputstack'],
classes[key], self.params)
if self.params['eotest'] is True:
self.makeEvenOddClasses('clstmp.lst', classes[key])
apDisplay.printMsg("\n")
reject.close()
keepfile.close()
os.remove('clstmp.lst')
# make 3d density file if specified:
if self.params['make3d'] is not None:
self.params['make3d'] = os.path.basename(self.params['make3d'])
outfile = os.path.join(self.params['rundir'],
self.params['make3d'])
apEMAN.make3d(self.params['stackname'],
outfile,
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d %s %s mask=%d norm" %
(outfile, outfile, self.params['mask']))
if self.params['eotest'] is True:
apEMAN.make3d(self.params['oddstack'],
"odd.mrc",
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.make3d(self.params['evenstack'],
"even.mrc",
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d odd.mrc even.mrc fsc=fsc.eotest")
if os.path.exists(outfile):
# run rmeasure
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
box = apVolume.getModelDimensions(outfile)
apDisplay.printMsg('inserting density into database')
symdata = apSymmetry.findSymmetry(self.params['sym'])
if not symdata:
apDisplay.printError('no symmetry associated with this model')
modq = appiondata.Ap3dDensityData()
modq['session'] = apStack.getSessionDataFromStackId(
self.params['stackid'])
modq['name'] = self.params['make3d']
modq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
modq['boxsize'] = box
modq['mask'] = self.params['mask']
modq['pixelsize'] = apix
fscres = apRecon.getResolutionFromFSCFile('fsc.eotest',
box,
apix,
msg=True)
modq['resolution'] = fscres
modq['rmeasure'] = apRecon.runRMeasure(apix, outfile)
modq['md5sum'] = apFile.md5sumfile(outfile)
modq['maxjump'] = self.params['avgjump']
modq['sigma'] = self.params['sigma']
modq['hard'] = self.params['hard']
modq['symmetry'] = symdata
modq['refineIter'] = self.params['refineiter']
if self.params['commit'] is True:
modq.insert()
apChimera.filterAndChimera(outfile,
res=fscres,
apix=apix,
box=box,
chimtype='snapshot',
zoom=self.params['zoom'],
sym=self.params['sym'],
mass=self.params['mass'])
else:
apDisplay.printError(
'no 3d volume was generated - check the class averages:')
apDisplay.printError(self.params['stackname'])
apDisplay.printError(
'hard may be set too high, or avg euler jump set too low for the # of particles'
)
stackstr = str(stackdata.dbid)
reconstr = str(self.params['reconid'])
apDisplay.printColor(
"Make a new stack with only non-jumpers:\n" +
"subStack.py --projectid=" + str(self.params['projectid']) +
" -s " + stackstr + " \\\n " + " -k " +
os.path.join(self.params['rundir'], "keep.lst") + " \\\n " +
" -d 'recon " + reconstr + " sitters' -n sitters" + reconstr +
" -C ", "purple")
def start(self):
self.params["outputstack"] = os.path.join(self.params["rundir"], self.params["stackname"])
particles, self.params["refineiter"] = getParticleInfo(self.params["reconid"], self.params["iter"])
stackdata = particles[0]["particle"]["stack"]
stack = os.path.join(stackdata["path"]["path"], stackdata["name"])
classes, cstats = determineClasses(particles)
rejectlst = []
if self.params["sigma"] is not None:
cutoff = cstats["meanquality"] + self.params["sigma"] * cstats["stdquality"]
apDisplay.printMsg("Cutoff = " + str(cutoff))
rejectlst = self.removePtclsByQualityFactor(particles, rejectlst, cutoff)
if self.params["avgjump"] is not None:
rejectlst = self.removePtclsByJumps(particles, rejectlst)
if self.params["rejectlst"]:
rejectlst = removePtclsByLst(rejectlst, self.params)
classkeys = classes.keys()
classkeys.sort()
classnum = 0
totalptcls = 0
keepfile = open("keep.lst", "w")
keepfile.write("#LST\n")
reject = open("reject.lst", "w")
reject.write("#LST\n")
apDisplay.printMsg("Processing classes")
# loop through classes
for key in classkeys:
# file to hold particles of this class
clsfile = open("clstmp.lst", "w")
clsfile.write("#LST\n")
classnum += 1
if classnum % 10 == 1:
apDisplay.printMsg(str(classnum) + " of " + (str(len(classkeys))))
images = EMAN.EMData()
# loop through particles in class
nptcls = 0
for ptcl in classes[key]["particles"]:
if ptcl["mirror"]:
mirror = 1
else:
mirror = 0
rot = ptcl["euler3"]
rot = rot * math.pi / 180
if ptcl["particle"]["particleNumber"] not in rejectlst:
l = "%d\t%s\t%f,\t%f,%f,%f,%d\n" % (
ptcl["particle"]["particleNumber"] - 1,
stack,
ptcl["quality_factor"],
rot,
ptcl["shiftx"],
ptcl["shifty"],
mirror,
)
keepfile.write(l)
clsfile.write(l)
totalptcls += 1
nptcls += 1
else:
reject.write(
"%d\t%s\t%f,\t%f,%f,%f,%d\n"
% (
ptcl["particle"]["particleNumber"] - 1,
stack,
ptcl["quality_factor"],
rot,
ptcl["shiftx"],
ptcl["shifty"],
mirror,
)
)
# if ptcl['quality_factor']>cstats['meanquality']+3*cstats['stdquality']:
# high.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1,
# stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror))
clsfile.close()
if nptcls < 1:
continue
if self.params["skipavg"] is False:
makeClassAverages("clstmp.lst", self.params["outputstack"], classes[key], self.params)
if self.params["eotest"] is True:
self.makeEvenOddClasses("clstmp.lst", classes[key])
apDisplay.printMsg("\n")
reject.close()
keepfile.close()
os.remove("clstmp.lst")
# make 3d density file if specified:
if self.params["make3d"] is not None:
self.params["make3d"] = os.path.basename(self.params["make3d"])
outfile = os.path.join(self.params["rundir"], self.params["make3d"])
apEMAN.make3d(
self.params["stackname"],
outfile,
sym=self.params["sym"],
mode=self.params["mode"],
hard=self.params["hard"],
)
apEMAN.executeEmanCmd("proc3d %s %s mask=%d norm" % (outfile, outfile, self.params["mask"]))
if self.params["eotest"] is True:
apEMAN.make3d(
self.params["oddstack"],
"odd.mrc",
sym=self.params["sym"],
mode=self.params["mode"],
hard=self.params["hard"],
)
apEMAN.make3d(
self.params["evenstack"],
"even.mrc",
sym=self.params["sym"],
mode=self.params["mode"],
hard=self.params["hard"],
)
apEMAN.executeEmanCmd("proc3d odd.mrc even.mrc fsc=fsc.eotest")
if os.path.exists(outfile):
# run rmeasure
apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
box = apVolume.getModelDimensions(outfile)
apDisplay.printMsg("inserting density into database")
symdata = apSymmetry.findSymmetry(self.params["sym"])
if not symdata:
apDisplay.printError("no symmetry associated with this model")
modq = appiondata.Ap3dDensityData()
modq["session"] = self.params["sessionname"]
modq["name"] = self.params["make3d"]
modq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
modq["boxsize"] = box
modq["mask"] = self.params["mask"]
modq["pixelsize"] = apix
fscres = apRecon.getResolutionFromFSCFile("fsc.eotest", box, apix, msg=True)
modq["resolution"] = fscres
modq["rmeasure"] = apRecon.runRMeasure(apix, outfile)
modq["md5sum"] = apFile.md5sumfile(outfile)
modq["maxjump"] = self.params["avgjump"]
modq["sigma"] = self.params["sigma"]
modq["hard"] = self.params["hard"]
modq["symmetry"] = symdata
modq["refineIter"] = self.params["refineiter"]
if self.params["commit"] is True:
modq.insert()
apChimera.filterAndChimera(
outfile,
res=fscres,
apix=apix,
box=box,
chimtype="snapshot",
zoom=self.params["zoom"],
sym=self.params["sym"],
mass=self.params["mass"],
)
else:
apDisplay.printError("no 3d volume was generated - check the class averages:")
apDisplay.printError(self.params["stackname"])
apDisplay.printError("hard may be set too high, or avg euler jump set too low for the # of particles")
stackstr = str(stackdata.dbid)
reconstr = str(self.params["reconid"])
apDisplay.printColor(
"Make a new stack with only non-jumpers:\n"
+ "subStack.py --projectid="
+ str(self.params["projectid"])
+ " -s "
+ stackstr
+ " \\\n "
+ " -k "
+ os.path.join(self.params["rundir"], "keep.lst")
+ " \\\n "
+ " -d 'recon "
+ reconstr
+ " sitters' -n sitters"
+ reconstr
+ " -C ",
"purple",
)
apDisplay.printWarning(
"Running Chimera Snapshot with resolution: %d " % resolution)
# TODO: need to work this out.
# symmetry was hard coded to 'c1'. why?
try:
symmetry = self.runparams['symmetry']
except Exception, e:
symmetry = self.runparams['symmetry'].split()[0]
symmetry = apSymmetry.findSymmetry(symmetry)
apChimera.filterAndChimera(volume,
resolution,
self.runparams['apix'],
self.runparams['boxsize'],
'snapshot',
self.params['contour'],
self.params['zoom'],
sym=symmetry,
mass=self.params['mass'])
#=====================
def calculateEulerJumpsAndGoodBadParticles(self, uploadIterations):
''' calculate euler jumps for entire recon, currently based on EMAN ZXZ convention '''
if self.params['commit'] is True:
reconrunid = self.refinerunq.dbid
### make table entries of good-bad particles
apRecon.setGoodBadParticlesFromReconId(reconrunid)
def insertIteration(self, iteration):
refineparamsq=appiondata.ApEmanRefineIterData()
refineparamsq['ang']=iteration['ang']
refineparamsq['lpfilter']=iteration['lpfilter']
refineparamsq['hpfilter']=iteration['hpfilter']
refineparamsq['pad']=iteration['pad']
refineparamsq['EMAN_maxshift']=iteration['maxshift']
refineparamsq['EMAN_hard']=iteration['hard']
refineparamsq['EMAN_classkeep']=iteration['classkeep']
refineparamsq['EMAN_classiter']=iteration['classiter']
refineparamsq['EMAN_filt3d']=iteration['filt3d']
refineparamsq['EMAN_shrink']=iteration['shrink']
refineparamsq['EMAN_euler2']=iteration['euler2']
refineparamsq['EMAN_xfiles']=iteration['xfiles']
refineparamsq['EMAN_median']=iteration['median']
refineparamsq['EMAN_phasecls']=iteration['phasecls']
refineparamsq['EMAN_fscls']=iteration['fscls']
refineparamsq['EMAN_refine']=iteration['refine']
refineparamsq['EMAN_goodbad']=iteration['goodbad']
refineparamsq['EMAN_perturb']=iteration['perturb']
refineparamsq['MsgP_cckeep']=iteration['msgpasskeep']
refineparamsq['MsgP_minptls']=iteration['msgpassminp']
#create Chimera snapshots
fscfile = os.path.join(self.params['rundir'], "fsc.eotest."+iteration['num'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix'])
if self.params['snapfilter']:
halfres = self.params['snapfilter']
volumeDensity = 'threed.'+iteration['num']+'a.mrc'
volDensPath = os.path.join(self.params['rundir'], volumeDensity)
apChimera.filterAndChimera(volDensPath, halfres, self.params['apix'],
self.params['boxsize'], 'snapshot', self.params['contour'], self.params['zoom'],
sym=iteration['sym']['eman_name'], mass=self.params['mass'])
## uncommment this for chimera image only runs...
if self.params['chimeraonly'] is True:
return
# insert resolution data
if halfres != True:
resData = self.getResolutionData(iteration)
else:
apDisplay.printWarning("resolution reported as nan, not committing results to database")
return
if self.params['package']== 'EMAN':
refineclassavg='classes_eman.'+iteration['num']+'.img'
postrefineclassavg=None
elif self.params['package']== 'EMAN/SpiCoran':
refineclassavg='classes_eman.'+iteration['num']+'.img'
postrefineclassavg='classes_coran.'+iteration['num']+'.img'
elif self.params['package']== 'EMAN/MsgP':
refineclassavg='classes_eman.'+iteration['num']+'.img'
postrefineclassavg='classes_msgp.'+iteration['num']+'.img'
else:
apDisplay.printError("Refinement Package Not Valid")
# insert refinement results
refineq = appiondata.ApRefineIterData()
refineq['refineRun'] = self.params['refineRun']
refineq['emanParams'] = refineparamsq
refineq['iteration'] = iteration['num']
refineq['resolution'] = resData
refineq['rMeasure'] = self.getRMeasureData(iteration)
refineq['mask'] = iteration['mask']
refineq['imask'] = iteration['imask']
refineq['symmetry']=iteration['sym']
refineq['exemplar'] = False
classvar = 'classes.'+iteration['num']+'.var.img'
refineq['refineClassAverages'] = refineclassavg
refineq['postRefineClassAverages'] = postrefineclassavg
if classvar in self.params['classvars']:
refineq['classVariance'] = classvar
if volumeDensity in self.params['volumes']:
refineq['volumeDensity'] = volumeDensity
apDisplay.printMsg("inserting Refinement Data into database")
if self.params['commit'] is True:
refineq.insert()
else:
apDisplay.printWarning("not committing results to database")
#insert FSC data
fscfile = os.path.join(self.params['rundir'], "fsc.eotest."+iteration['num'])
self.insertFSC(fscfile, refineq, self.params['commit'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix'])
apDisplay.printColor("FSC 0.5 Resolution: "+str(halfres), "cyan")
# get projections eulers for iteration:
eulers = self.getEulersFromProj(iteration['num'])
# get list of bad particles for this iteration
badprtls = self.readParticleLog(self.params['rundir'], iteration['num'])
# expand cls.*.tar into temp file
clsf=os.path.join(self.params['rundir'], "cls."+iteration['num']+".tar")
#print "reading",clsf
clstar=tarfile.open(clsf)
clslist=clstar.getmembers()
clsnames=clstar.getnames()
#print "extracting",clsf,"into temp directory"
for clsfile in clslist:
clstar.extract(clsfile,self.params['tmpdir'])
clstar.close()
# for each class, insert particle alignment info into database
apDisplay.printColor("Inserting Particle Classification Data for "
+str(len(clsnames))+" classes", "magenta")
t0 = time.time()
for cls in clsnames:
self.insertRefineParticleData(cls, iteration, eulers, badprtls, refineq, len(clsnames))
apDisplay.printColor("\nFinished in "+apDisplay.timeString(time.time()-t0), "magenta")
# remove temp directory
for file in os.listdir(self.params['tmpdir']):
os.remove(os.path.join(self.params['tmpdir'],file))
os.rmdir(self.params['tmpdir'])
#create euler freq map
apDisplay.printMsg("creating euler frequency map")
refrunid = int(self.params['refineRun'].dbid)
iternum = int(iteration['num'])
if self.params['package'] != 'EMAN':
postrefine = True
else:
postrefine = False
apEulerDraw.createEulerImages(refrunid, iternum, path=self.params['rundir'], postrefine=postrefine)
return