def correctCTFToDocFile(self, indocfile):
apDisplay.printMsg("Correcting CTF applied to particles")
inf = open(indocfile, 'r')
cmdlist = []
partnum = 0
for line in inf:
### get filename
partnum += 1
sline = line.strip()
if not sline:
continue
filename = sline.split()[0]
### correct CTF using ACE2
ace2cmd = (self.ace2correct+" -img %s -kv %d -cs %.3f -apix %.3f -df %.9f,%.9f,%.3f -wiener 0.1 -ampc %.3f"
%(filename, self.params['kv'], self.params['cs'], self.params['apix'],
self.deflist1c[partnum-1], self.deflist2c[partnum-1], self.params['astigmatism'], self.params['amp']))
cmdlist.append(ace2cmd)
numpart = partnum
inf.close()
### thread the commands
t0 = time.time()
apThread.threadCommands(cmdlist)
timeper = (time.time()-t0)/float(numpart)
apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green")
apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green")
### check for the files, write to doc file
inf = open(indocfile, 'r')
outdocfile = os.path.splitext(indocfile)[0]+".correct.lst"
outf = open(outdocfile, 'w')
for line in inf:
### get filename
filename = line.strip().split()[0]
newfile = filename+".corrected.mrc"
if not os.path.isfile(newfile):
apDisplay.printError("Ace 2 failed")
outf.write(newfile+"\t1\n")
inf.close()
outf.close()
return outdocfile
def start(self):
### get volume files
volumefiles = glob.glob(self.params['volumes'])
if not volumefiles:
apDisplay.printError("Could not find volumes, %s" %
(self.params['volumes']))
### make list of all alignments to run
cmdlist = []
alignfiles = []
for volfile in volumefiles:
alignfile = "align-" + os.path.basename(volfile)
alignfiles.append(alignfile)
emancmd = "align3d %s %s %s " % (self.params['reference'], volfile,
alignfile)
if self.params['slow'] is True:
emancmd += "slow "
if self.params['noshrink'] is True:
emancmd += "noshrink "
#print emancmd
cmdlist.append(emancmd)
### run several alignment commands in parallel
apThread.threadCommands(cmdlist)
### average volumes together
ref = mrc.read(self.params['reference'])
average = numpy.zeros(ref.shape, dtype=numpy.float32)
del ref
count = 0
for alignfile in alignfiles:
if not os.path.isfile(alignfile):
apDisplay.printWarning("aligned volume not found: %s" %
(alignfile))
aligned = mrc.read(alignfile)
count += 1
### this assume all aligned volume have same box size
average += aligned
del aligned
### save average
average /= count
avgfile = os.path.abspath(self.params['average'])
mrc.write(average, avgfile)
apDisplay.printMsg("Wrote average file: " + avgfile)
def singleNodeRun(self, iternum):
"""
single node run
"""
### single node run
apDisplay.printMsg("Single node run, iteration %d" % (iternum))
### create individual processor jobs
stackbase = os.path.splitext(os.path.basename(self.refinestackfile))[0]
self.stackfile = "../../%s" % (stackbase)
procjobfiles = self.createMultipleJobs(iternum)
### convert job files to commands
procjobcmds = []
for procjobfile in procjobfiles:
procjobcmd = "sh " + procjobfile
procjobcmds.append(procjobcmd)
### run individual processor jobs
t0 = time.time()
apThread.threadCommands(procjobcmds,
nproc=self.params['nproc'],
pausetime=30.0)
apDisplay.printColor(
"Refinement complete in %s" %
(apDisplay.timeString(time.time() - t0)), "green")
### create combine processor jobs
stackbase = os.path.splitext(os.path.basename(self.reconstackfile))[0]
self.stackfile = "../%s" % (stackbase)
combinejobfile = self.combineMultipleJobs(iternum)
### run combine processor jobs
t0 = time.time()
proc = subprocess.Popen("sh " + combinejobfile, shell=True)
proc.wait()
apDisplay.printColor(
"Volume complete in %s" % (apDisplay.timeString(time.time() - t0)),
"green")
if apFile.fileSize('iter%03d.img' % (iternum)) < 100:
apDisplay.printError("Failed to generate volume for iter %d" %
(iternum))
return
def correctCTFToDocFile(self, indocfile):
apDisplay.printMsg("Correcting CTF applied to particles")
inf = open(indocfile, 'r')
cmdlist = []
partnum = 0
for line in inf:
### get filename
partnum += 1
sline = line.strip()
if not sline:
continue
filename = sline.split()[0]
### correct CTF using ACE2
ace2cmd = (self.ace2correct+" -img %s -kv %d -cs %.3f -apix %.3f -df %.9f,%.9f,%.3f -wiener 0.1 -ampc %.3f"
%(filename, self.params['kv'], self.params['cs'], self.params['apix'],
self.deflist1c[partnum-1], self.deflist2c[partnum-1], self.params['astigmatism'], self.params['amp']))
cmdlist.append(ace2cmd)
numpart = partnum
inf.close()
### thread the commands
t0 = time.time()
apThread.threadCommands(cmdlist)
timeper = (time.time()-t0)/float(numpart)
apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green")
apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green")
### check for the files, write to doc file
inf = open(indocfile, 'r')
outdocfile = os.path.splitext(indocfile)[0]+".correct.lst"
outf = open(outdocfile, 'w')
for line in inf:
### get filename
filename = line.strip().split()[0]
newfile = filename+".corrected.mrc"
if not os.path.isfile(newfile):
apDisplay.printError("Ace 2 failed")
outf.write(newfile+"\t1\n")
inf.close()
outf.close()
return outdocfile
def start(self):
### get volume files
volumefiles = glob.glob(self.params['volumes'])
if not volumefiles:
apDisplay.printError("Could not find volumes, %s"%(self.params['volumes']))
### make list of all alignments to run
cmdlist = []
alignfiles = []
for volfile in volumefiles:
alignfile = "align-"+os.path.basename(volfile)
alignfiles.append(alignfile)
emancmd = "align3d %s %s %s "%(self.params['reference'], volfile, alignfile)
if self.params['slow'] is True:
emancmd += "slow "
if self.params['noshrink'] is True:
emancmd += "noshrink "
#print emancmd
cmdlist.append(emancmd)
### run several alignment commands in parallel
apThread.threadCommands(cmdlist)
### average volumes together
ref = mrc.read(self.params['reference'])
average = numpy.zeros(ref.shape, dtype=numpy.float32)
del ref
count = 0
for alignfile in alignfiles:
if not os.path.isfile(alignfile):
apDisplay.printWarning("aligned volume not found: %s"%(alignfile))
aligned = mrc.read(alignfile)
count += 1
### this assume all aligned volume have same box size
average += aligned
del aligned
### save average
average /= count
avgfile = os.path.abspath(self.params['average'])
mrc.write(average, avgfile)
apDisplay.printMsg("Wrote average file: "+avgfile)
def singleNodeRun(self, iternum):
"""
single node run
"""
### single node run
apDisplay.printMsg("Single node run, iteration %d"%(iternum))
### create individual processor jobs
stackbase = os.path.splitext(os.path.basename(self.refinestackfile))[0]
self.stackfile = "../../%s"%(stackbase)
procjobfiles = self.createMultipleJobs(iternum)
### convert job files to commands
procjobcmds = []
for procjobfile in procjobfiles:
procjobcmd = "sh "+procjobfile
procjobcmds.append(procjobcmd)
### run individual processor jobs
t0 = time.time()
apThread.threadCommands(procjobcmds, nproc=self.params['nproc'], pausetime=30.0)
apDisplay.printColor("Refinement complete in %s"%(apDisplay.timeString(time.time()-t0)), "green")
### create combine processor jobs
stackbase = os.path.splitext(os.path.basename(self.reconstackfile))[0]
self.stackfile = "../%s"%(stackbase)
combinejobfile = self.combineMultipleJobs(iternum)
### run combine processor jobs
t0 = time.time()
proc = subprocess.Popen("sh "+combinejobfile, shell=True)
proc.wait()
apDisplay.printColor("Volume complete in %s"%(apDisplay.timeString(time.time()-t0)), "green")
if apFile.fileSize('iter%03d.img'%(iternum)) < 100:
apDisplay.printError("Failed to generate volume for iter %d"%(iternum))
return
def start(self):
'''
OptiMod script
'''
############### define short-hand parameter names ############
refine = True
rundir = self.params['rundir']
box = self.params['boxsize']
rbox = self.params['refineboxsize']
apix = self.params['apix']
rapix = self.params['refineapix']
npart = self.params['numpart']
nvol = self.params['num_volumes']
# sym = self.params['sym']
sym = "c1"
psym = self.params['presumed_sym']
first = self.params['firstimage']
nws = self.params['non_weighted_sequence']
threes = self.params['threes']
lmask = self.params['linmask']
asqfilt = self.params['asqfilt']
anginc = self.params['ang_inc']
keep_ordered = self.params['keep_ordered_num']
hamwin = self.params['ham_win']
useEMAN1 = self.params['useEMAN1']
ipv = self.params['images_per_volume']
lp = self.params['3d_lpfilt']
nref = self.params['nref']
PCA = self.params['PCA']
neigens = self.params['numeigens']
recalc = self.params['recalculate']
preftype = self.params['preftype']
mrad = self.params['mask_radius']
irad = self.params['inner_radius']
orad = self.params['outer_radius']
tnproc = self.params['threadnproc']
nproc = self.params['nproc']
oldavgs = self.params['oldavgs']
avgs = os.path.join(self.params['rundir'], self.params['avgs'])
start = self.params['start_at']
############# copy to working directory ############
if not os.path.isfile(os.path.join(avgs)[:-4]+".hed"):
shutil.copyfile(os.path.join(oldavgs[:-4]+".hed"), avgs[:-4]+".hed")
if not os.path.isfile(os.path.join(avgs)[:-4]+".img"):
shutil.copyfile(os.path.join(oldavgs[:-4]+".img"), avgs[:-4]+".img")
if self.params['ravgs'] is not None:
ravgs = os.path.join(self.params['rundir'], self.params['ravgs'])
if not os.path.isfile(self.params['ravgs'][:-4]+".hed"):
shutil.copy(self.params['oldravgs'][:-4]+".hed",
os.path.join(self.params['ravgs'][:-4]+".hed"))
if not os.path.isfile(self.params['ravgs'][:-4]+".img"):
shutil.copy(self.params['oldravgs'][:-4]+".img",
os.path.join(self.params['ravgs'][:-4]+".img"))
### Euler jumper assessment does not make sense when refine images are different
ejassess = False
else:
ravgs = avgs
ejassess = True
########### scale & prealign class averages, if specified #########
if self.params['scale'] is True:
emancmd = "proc2d %s %s_scaled.img scale=%.3f clip=%i,%i edgenorm" \
% (avgs, avgs[:-4], self.scalefactor, 64, 64)
avgs = avgs[:-4]+"_scaled.img"
if start == "none":
while os.path.isfile(avgs):
apFile.removeStack(avgs)
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printColor("skipping stack scaling", "cyan")
if self.imagicroot is not None and useEMAN1 is False:
apIMAGIC.takeoverHeaders(avgs, npart, box)
if self.imagicroot is not None:
if self.params['prealign'] is True:
if start == "none":
avgs = apIMAGIC.prealignClassAverages(rundir, avgs)
apIMAGIC.checkLogFileForErrors(
os.path.join(rundir, "prealignClassAverages.log"))
### select between EMAN1 and IMAGIC for common lines
if self.imagicroot is not None and useEMAN1 is False:
commonlinesdir = os.path.join(rundir, "angular_reconstitution")
else:
commonlinesdir = os.path.join(rundir, "cross_common_lines")
### make links & calculate CCC matrix for 2D averages
apDisplay.printColor("Calculating similarity matrix", "cyan")
ccc_matrix = apCommonLines.calculate_ccc_matrix_2d(avgs)
clsavgs = os.path.split(avgs)[1][:-4]
if not os.path.isdir(commonlinesdir):
os.mkdir(commonlinesdir)
if os.path.islink(os.path.join(commonlinesdir, clsavgs+".hed")):
os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".hed"))
os.symlink(os.path.join(rundir, clsavgs+".hed"),
os.path.join(commonlinesdir, clsavgs+".hed"))
if os.path.islink(os.path.join(commonlinesdir, clsavgs+".img")):
os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".img"))
os.symlink(os.path.join(rundir, clsavgs+".img"),
os.path.join(commonlinesdir, clsavgs+".img"))
################## create multiple 3d0s #####################
if start == "none":
if self.imagicroot is not None and useEMAN1 is False:
cmdlist = []
seqfile = open(os.path.join(rundir,
"sequences_for_angular_reconstitution.dat"), "w")
apDisplay.printColor("Running multiple raw volume calculations", "cyan")
for i in range(nvol):
sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart,
ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
apCommonLines.check_for_duplicates_in_sequence(sequence)
seqfile.write(str(sequence)+"\n")
### create IMAGIC batch file for each model & append them to be threaded
batchfile = apCommonLines.imagic_batch_file(sequence, i+1, avgs, sym,
asqfilt, lmask, apix, box, anginc, keep_ordered, hamwin, lp,
threes=threes, do_not_remove=False)
proc = subprocess.Popen('chmod 755 '+batchfile, shell=True)
proc.wait()
cmdlist.append(batchfile)
os.chdir(rundir)
seqfile.close()
apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)
### check for errors after execution
for i in range(nvol):
apIMAGIC.checkLogFileForErrors(
os.path.join(commonlinesdir, "3d"+str(i+1)+".log"))
else:
### use EMAN1 cross-common lines
cmdlist = []
seqfile = open(os.path.join(rundir, "sequences_for_cross_common_lines.dat"), "w")
for i in range(nvol):
sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart,
ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
apCommonLines.check_for_duplicates_in_sequence(sequence)
seqfile.write(str(sequence)+"\n")
vol_avgfile = os.path.join(commonlinesdir, "vol_%d_averages.hed" % (i+1))
vol_seqfile = os.path.join(commonlinesdir, "vol_%d_averages.txt" % (i+1))
vsf = open(vol_seqfile, "w")
for j in range(ipv):
vsf.write("%d\n" % sequence[j])
vsf.close()
tmpdir = os.path.join(commonlinesdir, "tmp%d" % (i+1))
fullcmd = "proc2d %s %s list=%s ; " % (avgs, vol_avgfile, vol_seqfile)
fullcmd+= "rm -rf %s ; mkdir %s ; cd %s ; " % (tmpdir, tmpdir, tmpdir)
fullcmd+= "startAny %s sym=c1 proc=1 rounds=2 mask=%d ; " \
% (vol_avgfile, mrad/apix)
fullcmd+= "mv threed.0a.mrc ../threed_%d.mrc ; " % (i+1)
fullcmd+= "mv CCL.hed ../CCL_%d.hed ; " % (i+1)
fullcmd+= "mv CCL.img ../CCL_%d.img ; " % (i+1)
cmdlist.append(fullcmd)
seqfile.close()
apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)
for i in range(nvol):
shutil.rmtree(os.path.join(commonlinesdir, "tmp%d" % (i+1)))
else:
apDisplay.printColor("skipping common lines volume calculations", "cyan")
############### move 3-D models volume directory ################
### create volume directory
volumedir = os.path.join(rundir, "volumes")
if not os.path.isdir(volumedir):
os.mkdir(volumedir)
volumes = {}
cmds = []
if start == "none":
apDisplay.printColor("moving volumes for Xmipp 3-D Maximum Likelihood", "cyan")
for i in range(nvol):
if useEMAN1 is False:
volume1 = os.path.join(commonlinesdir, "3d%d_ordered%d_filt.vol" % (i+1,i+1))
else:
volume1 = os.path.join(commonlinesdir, "threed_%d.mrc" % (i+1))
volume2 = os.path.join(volumedir, "3d%d.vol" % (i+1))
if start == "none":
if useEMAN1 is False:
shutil.move(volume1, volume2)
else:
cmds.append("proc3d %s %s spidersingle" % (volume1, volume2))
volumes[(i+1)] = volume2
apThread.threadCommands(cmds, nproc=tnproc)
#################### align 3-D models ##################
if start == "3D_align" or start == "none":
# if start != "3D_assess" and start != "3D_refine":
### run Maximum Likelihood 3-D alignment & align resulting volumes
apDisplay.printColor("Running Xmipp maximum likelihood 3-D alignment", "cyan")
vol_doc_file, alignref = apCommonLines.xmipp_max_like_3d_align(
volumes, nproc, tnproc, nref)
alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)
apDisplay.printColor("Aligning volumes based on 3-D ML parameters", "cyan")
apCommonLines.align_volumes(alignparams, alignref, nvol, tnproc, apix)
else:
apDisplay.printColor("skipping 3D alignment", "cyan")
vol_doc_file, alignref = apCommonLines.findAlignmentParams(
os.path.join(rundir,"max_like_alignment"), nref)
alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)
##################### Principal Component Analysis ##############
apDisplay.printColor("Calculating inter-volume similarity", "cyan")
aligned_volumes = {}
for i in range(nvol):
aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.vol" % (i+1))
# aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.mrc" % (i+1))
if PCA is True:
simfile, sim_matrix = apCommonLines.runPrincipalComponentAnalysis(
aligned_volumes, nvol, neigens, box, apix, recalculate=recalc)
else:
simfile, sim_matrix = apCommonLines.calculate_ccc_matrix_3d(
aligned_volumes, nvol)
################ 3-D affinity propagation #############
### 3-D Affinity Propagation
apDisplay.printColor("Averaging volumes with Affinity Propagation", "cyan")
preffile = apCommonLines.set_preferences(sim_matrix, preftype, nvol)
classes = apCommonLines.run_affinity_propagation(aligned_volumes, simfile,
preffile, box, apix)
######### refine volumes using Xmipp projection matching #########
if start == "3D_align" or start == "3D_refine" or start == "none":
# if start != "3D_assess":
if not os.path.isdir("refinement"):
os.mkdir("refinement")
os.chdir("refinement")
apXmipp.breakupStackIntoSingleFiles(ravgs)
xmippcmd = "xmipp_normalize -i partlist.sel -method OldXmipp"
apParam.runCmd(xmippcmd, "Xmipp")
for i in classes.keys():
emancmd = "proc3d %s %s scale=%.3f clip=%d,%d,%d mask=%s spidersingle" \
% (os.path.join(rundir, "%d.mrc" % i), "%d.vol" % i, \
(1/self.scalefactor), rbox, rbox, rbox, (mrad / rapix))
apParam.runCmd(emancmd, "EMAN")
apCommonLines.refine_volume(rundir, ravgs, i, mrad, irad, orad, rapix, nproc)
emancmd = "proc3d %s %s apix=%.3f" \
% (os.path.join("refine_%d" % i, "3d%d_refined.vol" % i), \
os.path.join(rundir, "%d_r.mrc" % i), rapix)
apParam.runCmd(emancmd, "EMAN")
os.chdir(rundir)
else:
apDisplay.printColor("skipping 3D refinement", "cyan")
################# model evaluation ################
if start == "3D_align" or start == "3D_refine" or start == "3D_assess" or start == "none":
# if start == "none" or start=="3D_assess":
### final model assessment
try:
euler_array = apCommonLines.getEulerValuesForModels(alignparams,
rundir, nvol, npart, threes=threes)
except:
euler_array = None
ejassess = False
if refine is True:
apCommonLines.assess_3Dclass_quality2(rundir, aligned_volumes, sim_matrix,
classes, euler_array, box, rbox, rapix, ravgs, psym, npart, ejassess=ejassess)
else:
apCommonLines.assess_3Dclass_quality(rundir, aligned_volumes, sim_matrix,
classes, euler_array, box, apix, avgs, psym, npart, ejassess=ejassess)
apCommonLines.combineMetrics("final_model_stats.dat",
"final_model_stats_sorted_by_Rcrit.dat", **{"CCPR":(1,1)})
################## upload & cleanup ##################
### upload to database, if specified
self.upload()
### make chimera snapshots
if refine is True:
for i in classes.keys():
if self.params['mass'] is not None:
apChimera.filterAndChimera(
os.path.join(self.params['rundir'], "%d_r.mrc" % i),
res=self.params['3d_lpfilt'],
apix=self.params['refineapix'],
box=self.params['refineboxsize'],
chimtype="snapshot",
contour=2,
zoom=1,
sym="c1",
color="gold",
mass=self.params['mass']
)
### cleanup
snapshots = glob.glob("*.png")
mtlfiles = glob.glob("*.mtl")
objfiles = glob.glob("*.obj")
pyfiles = glob.glob("*mrc.py")
for file in mtlfiles:
os.remove(file)
for file in objfiles:
os.remove(file)
for file in pyfiles:
os.remove(file)
if not os.path.isdir("snapshots"):
os.mkdir("snapshots")
for s in snapshots:
shutil.move(s, os.path.join("snapshots", s))
def runMultipleFreeHandTest(self, commandlist):
apThread.threadCommands(commandlist,
nproc=self.params['nproc'],
pausetime=1.0)
def runMultipleFreeHandTest(self, commandlist):
apThread.threadCommands(commandlist, nproc=self.params["nproc"], pausetime=1.0)