def start(self):
'''
OptiMod script
'''
############### define short-hand parameter names ############
refine = True
rundir = self.params['rundir']
box = self.params['boxsize']
rbox = self.params['refineboxsize']
apix = self.params['apix']
rapix = self.params['refineapix']
npart = self.params['numpart']
nvol = self.params['num_volumes']
# sym = self.params['sym']
sym = "c1"
psym = self.params['presumed_sym']
first = self.params['firstimage']
nws = self.params['non_weighted_sequence']
threes = self.params['threes']
lmask = self.params['linmask']
asqfilt = self.params['asqfilt']
anginc = self.params['ang_inc']
keep_ordered = self.params['keep_ordered_num']
hamwin = self.params['ham_win']
useEMAN1 = self.params['useEMAN1']
ipv = self.params['images_per_volume']
lp = self.params['3d_lpfilt']
nref = self.params['nref']
PCA = self.params['PCA']
neigens = self.params['numeigens']
recalc = self.params['recalculate']
preftype = self.params['preftype']
mrad = self.params['mask_radius']
irad = self.params['inner_radius']
orad = self.params['outer_radius']
tnproc = self.params['threadnproc']
nproc = self.params['nproc']
oldavgs = self.params['oldavgs']
avgs = os.path.join(self.params['rundir'], self.params['avgs'])
start = self.params['start_at']
############# copy to working directory ############
if not os.path.isfile(os.path.join(avgs)[:-4]+".hed"):
shutil.copyfile(os.path.join(oldavgs[:-4]+".hed"), avgs[:-4]+".hed")
if not os.path.isfile(os.path.join(avgs)[:-4]+".img"):
shutil.copyfile(os.path.join(oldavgs[:-4]+".img"), avgs[:-4]+".img")
if self.params['ravgs'] is not None:
ravgs = os.path.join(self.params['rundir'], self.params['ravgs'])
if not os.path.isfile(self.params['ravgs'][:-4]+".hed"):
shutil.copy(self.params['oldravgs'][:-4]+".hed",
os.path.join(self.params['ravgs'][:-4]+".hed"))
if not os.path.isfile(self.params['ravgs'][:-4]+".img"):
shutil.copy(self.params['oldravgs'][:-4]+".img",
os.path.join(self.params['ravgs'][:-4]+".img"))
### Euler jumper assessment does not make sense when refine images are different
ejassess = False
else:
ravgs = avgs
ejassess = True
########### scale & prealign class averages, if specified #########
if self.params['scale'] is True:
emancmd = "proc2d %s %s_scaled.img scale=%.3f clip=%i,%i edgenorm" \
% (avgs, avgs[:-4], self.scalefactor, 64, 64)
avgs = avgs[:-4]+"_scaled.img"
if start == "none":
while os.path.isfile(avgs):
apFile.removeStack(avgs)
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printColor("skipping stack scaling", "cyan")
if self.imagicroot is not None and useEMAN1 is False:
apIMAGIC.takeoverHeaders(avgs, npart, box)
if self.imagicroot is not None:
if self.params['prealign'] is True:
if start == "none":
avgs = apIMAGIC.prealignClassAverages(rundir, avgs)
apIMAGIC.checkLogFileForErrors(
os.path.join(rundir, "prealignClassAverages.log"))
### select between EMAN1 and IMAGIC for common lines
if self.imagicroot is not None and useEMAN1 is False:
commonlinesdir = os.path.join(rundir, "angular_reconstitution")
else:
commonlinesdir = os.path.join(rundir, "cross_common_lines")
### make links & calculate CCC matrix for 2D averages
apDisplay.printColor("Calculating similarity matrix", "cyan")
ccc_matrix = apCommonLines.calculate_ccc_matrix_2d(avgs)
clsavgs = os.path.split(avgs)[1][:-4]
if not os.path.isdir(commonlinesdir):
os.mkdir(commonlinesdir)
if os.path.islink(os.path.join(commonlinesdir, clsavgs+".hed")):
os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".hed"))
os.symlink(os.path.join(rundir, clsavgs+".hed"),
os.path.join(commonlinesdir, clsavgs+".hed"))
if os.path.islink(os.path.join(commonlinesdir, clsavgs+".img")):
os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".img"))
os.symlink(os.path.join(rundir, clsavgs+".img"),
os.path.join(commonlinesdir, clsavgs+".img"))
################## create multiple 3d0s #####################
if start == "none":
if self.imagicroot is not None and useEMAN1 is False:
cmdlist = []
seqfile = open(os.path.join(rundir,
"sequences_for_angular_reconstitution.dat"), "w")
apDisplay.printColor("Running multiple raw volume calculations", "cyan")
for i in range(nvol):
sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart,
ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
apCommonLines.check_for_duplicates_in_sequence(sequence)
seqfile.write(str(sequence)+"\n")
### create IMAGIC batch file for each model & append them to be threaded
batchfile = apCommonLines.imagic_batch_file(sequence, i+1, avgs, sym,
asqfilt, lmask, apix, box, anginc, keep_ordered, hamwin, lp,
threes=threes, do_not_remove=False)
proc = subprocess.Popen('chmod 755 '+batchfile, shell=True)
proc.wait()
cmdlist.append(batchfile)
os.chdir(rundir)
seqfile.close()
apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)
### check for errors after execution
for i in range(nvol):
apIMAGIC.checkLogFileForErrors(
os.path.join(commonlinesdir, "3d"+str(i+1)+".log"))
else:
### use EMAN1 cross-common lines
cmdlist = []
seqfile = open(os.path.join(rundir, "sequences_for_cross_common_lines.dat"), "w")
for i in range(nvol):
sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart,
ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
apCommonLines.check_for_duplicates_in_sequence(sequence)
seqfile.write(str(sequence)+"\n")
vol_avgfile = os.path.join(commonlinesdir, "vol_%d_averages.hed" % (i+1))
vol_seqfile = os.path.join(commonlinesdir, "vol_%d_averages.txt" % (i+1))
vsf = open(vol_seqfile, "w")
for j in range(ipv):
vsf.write("%d\n" % sequence[j])
vsf.close()
tmpdir = os.path.join(commonlinesdir, "tmp%d" % (i+1))
fullcmd = "proc2d %s %s list=%s ; " % (avgs, vol_avgfile, vol_seqfile)
fullcmd+= "rm -rf %s ; mkdir %s ; cd %s ; " % (tmpdir, tmpdir, tmpdir)
fullcmd+= "startAny %s sym=c1 proc=1 rounds=2 mask=%d ; " \
% (vol_avgfile, mrad/apix)
fullcmd+= "mv threed.0a.mrc ../threed_%d.mrc ; " % (i+1)
fullcmd+= "mv CCL.hed ../CCL_%d.hed ; " % (i+1)
fullcmd+= "mv CCL.img ../CCL_%d.img ; " % (i+1)
cmdlist.append(fullcmd)
seqfile.close()
apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)
for i in range(nvol):
shutil.rmtree(os.path.join(commonlinesdir, "tmp%d" % (i+1)))
else:
apDisplay.printColor("skipping common lines volume calculations", "cyan")
############### move 3-D models volume directory ################
### create volume directory
volumedir = os.path.join(rundir, "volumes")
if not os.path.isdir(volumedir):
os.mkdir(volumedir)
volumes = {}
cmds = []
if start == "none":
apDisplay.printColor("moving volumes for Xmipp 3-D Maximum Likelihood", "cyan")
for i in range(nvol):
if useEMAN1 is False:
volume1 = os.path.join(commonlinesdir, "3d%d_ordered%d_filt.vol" % (i+1,i+1))
else:
volume1 = os.path.join(commonlinesdir, "threed_%d.mrc" % (i+1))
volume2 = os.path.join(volumedir, "3d%d.vol" % (i+1))
if start == "none":
if useEMAN1 is False:
shutil.move(volume1, volume2)
else:
cmds.append("proc3d %s %s spidersingle" % (volume1, volume2))
volumes[(i+1)] = volume2
apThread.threadCommands(cmds, nproc=tnproc)
#################### align 3-D models ##################
if start == "3D_align" or start == "none":
# if start != "3D_assess" and start != "3D_refine":
### run Maximum Likelihood 3-D alignment & align resulting volumes
apDisplay.printColor("Running Xmipp maximum likelihood 3-D alignment", "cyan")
vol_doc_file, alignref = apCommonLines.xmipp_max_like_3d_align(
volumes, nproc, tnproc, nref)
alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)
apDisplay.printColor("Aligning volumes based on 3-D ML parameters", "cyan")
apCommonLines.align_volumes(alignparams, alignref, nvol, tnproc, apix)
else:
apDisplay.printColor("skipping 3D alignment", "cyan")
vol_doc_file, alignref = apCommonLines.findAlignmentParams(
os.path.join(rundir,"max_like_alignment"), nref)
alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)
##################### Principal Component Analysis ##############
apDisplay.printColor("Calculating inter-volume similarity", "cyan")
aligned_volumes = {}
for i in range(nvol):
aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.vol" % (i+1))
# aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.mrc" % (i+1))
if PCA is True:
simfile, sim_matrix = apCommonLines.runPrincipalComponentAnalysis(
aligned_volumes, nvol, neigens, box, apix, recalculate=recalc)
else:
simfile, sim_matrix = apCommonLines.calculate_ccc_matrix_3d(
aligned_volumes, nvol)
################ 3-D affinity propagation #############
### 3-D Affinity Propagation
apDisplay.printColor("Averaging volumes with Affinity Propagation", "cyan")
preffile = apCommonLines.set_preferences(sim_matrix, preftype, nvol)
classes = apCommonLines.run_affinity_propagation(aligned_volumes, simfile,
preffile, box, apix)
######### refine volumes using Xmipp projection matching #########
if start == "3D_align" or start == "3D_refine" or start == "none":
# if start != "3D_assess":
if not os.path.isdir("refinement"):
os.mkdir("refinement")
os.chdir("refinement")
apXmipp.breakupStackIntoSingleFiles(ravgs)
xmippcmd = "xmipp_normalize -i partlist.sel -method OldXmipp"
apParam.runCmd(xmippcmd, "Xmipp")
for i in classes.keys():
emancmd = "proc3d %s %s scale=%.3f clip=%d,%d,%d mask=%s spidersingle" \
% (os.path.join(rundir, "%d.mrc" % i), "%d.vol" % i, \
(1/self.scalefactor), rbox, rbox, rbox, (mrad / rapix))
apParam.runCmd(emancmd, "EMAN")
apCommonLines.refine_volume(rundir, ravgs, i, mrad, irad, orad, rapix, nproc)
emancmd = "proc3d %s %s apix=%.3f" \
% (os.path.join("refine_%d" % i, "3d%d_refined.vol" % i), \
os.path.join(rundir, "%d_r.mrc" % i), rapix)
apParam.runCmd(emancmd, "EMAN")
os.chdir(rundir)
else:
apDisplay.printColor("skipping 3D refinement", "cyan")
################# model evaluation ################
if start == "3D_align" or start == "3D_refine" or start == "3D_assess" or start == "none":
# if start == "none" or start=="3D_assess":
### final model assessment
try:
euler_array = apCommonLines.getEulerValuesForModels(alignparams,
rundir, nvol, npart, threes=threes)
except:
euler_array = None
ejassess = False
if refine is True:
apCommonLines.assess_3Dclass_quality2(rundir, aligned_volumes, sim_matrix,
classes, euler_array, box, rbox, rapix, ravgs, psym, npart, ejassess=ejassess)
else:
apCommonLines.assess_3Dclass_quality(rundir, aligned_volumes, sim_matrix,
classes, euler_array, box, apix, avgs, psym, npart, ejassess=ejassess)
apCommonLines.combineMetrics("final_model_stats.dat",
"final_model_stats_sorted_by_Rcrit.dat", **{"CCPR":(1,1)})
################## upload & cleanup ##################
### upload to database, if specified
self.upload()
### make chimera snapshots
if refine is True:
for i in classes.keys():
if self.params['mass'] is not None:
apChimera.filterAndChimera(
os.path.join(self.params['rundir'], "%d_r.mrc" % i),
res=self.params['3d_lpfilt'],
apix=self.params['refineapix'],
box=self.params['refineboxsize'],
chimtype="snapshot",
contour=2,
zoom=1,
sym="c1",
color="gold",
mass=self.params['mass']
)
### cleanup
snapshots = glob.glob("*.png")
mtlfiles = glob.glob("*.mtl")
objfiles = glob.glob("*.obj")
pyfiles = glob.glob("*mrc.py")
for file in mtlfiles:
os.remove(file)
for file in objfiles:
os.remove(file)
for file in pyfiles:
os.remove(file)
if not os.path.isdir("snapshots"):
os.mkdir("snapshots")
for s in snapshots:
shutil.move(s, os.path.join("snapshots", s))
return parser.parse_args()
def checkConflicts(options, args):
### metrics with values
if options.infile == None:
errmsg = "specify the input filename with quality metrics and associated values "
errmsg+= "e.g. --infile=final_model_stats.dat"
apDisplay.printError(errmsg)
if options.outfile == None:
errmsg = "specify the output filename with quality metrics and associated values "
errmsg+= "e.g. --outfile=final_model_stats_sorted_by_Rcrit.dat"
apDisplay.printError(errmsg)
if options.metrics == None:
errmsg = "please specify the desired metric(s) to combine, along with appropriate "
errmsg+= "weight and sign, for example: --metrics='CCPR,1,1 EJ,0.5,-1 FSC,0.1,-1'"
apDisplay.printError(errmsg)
return
##########################
if __name__ == "__main__":
options, args = setupParserOptions()
checkConflicts(options, args)
metrics = options.metrics.split()
kwargs = {}
for metric in metrics:
m, weight, sign = metric.split(",")
kwargs[m] = (float(weight),int(sign))
apCommonLines.combineMetrics(options.infile, options.outfile, **kwargs)
