def start(self): ''' OptiMod script ''' ############### define short-hand parameter names ############ refine = True rundir = self.params['rundir'] box = self.params['boxsize'] rbox = self.params['refineboxsize'] apix = self.params['apix'] rapix = self.params['refineapix'] npart = self.params['numpart'] nvol = self.params['num_volumes'] # sym = self.params['sym'] sym = "c1" psym = self.params['presumed_sym'] first = self.params['firstimage'] nws = self.params['non_weighted_sequence'] threes = self.params['threes'] lmask = self.params['linmask'] asqfilt = self.params['asqfilt'] anginc = self.params['ang_inc'] keep_ordered = self.params['keep_ordered_num'] hamwin = self.params['ham_win'] useEMAN1 = self.params['useEMAN1'] ipv = self.params['images_per_volume'] lp = self.params['3d_lpfilt'] nref = self.params['nref'] PCA = self.params['PCA'] neigens = self.params['numeigens'] recalc = self.params['recalculate'] preftype = self.params['preftype'] mrad = self.params['mask_radius'] irad = self.params['inner_radius'] orad = self.params['outer_radius'] tnproc = self.params['threadnproc'] nproc = self.params['nproc'] oldavgs = self.params['oldavgs'] avgs = os.path.join(self.params['rundir'], self.params['avgs']) start = self.params['start_at'] ############# copy to working directory ############ if not os.path.isfile(os.path.join(avgs)[:-4]+".hed"): shutil.copyfile(os.path.join(oldavgs[:-4]+".hed"), avgs[:-4]+".hed") if not os.path.isfile(os.path.join(avgs)[:-4]+".img"): shutil.copyfile(os.path.join(oldavgs[:-4]+".img"), avgs[:-4]+".img") if self.params['ravgs'] is not None: ravgs = os.path.join(self.params['rundir'], self.params['ravgs']) if not os.path.isfile(self.params['ravgs'][:-4]+".hed"): shutil.copy(self.params['oldravgs'][:-4]+".hed", os.path.join(self.params['ravgs'][:-4]+".hed")) if not os.path.isfile(self.params['ravgs'][:-4]+".img"): shutil.copy(self.params['oldravgs'][:-4]+".img", os.path.join(self.params['ravgs'][:-4]+".img")) ### Euler jumper assessment does not make sense when refine images are different ejassess = False else: ravgs = avgs ejassess = True ########### scale & prealign class averages, if specified ######### if self.params['scale'] is True: emancmd = "proc2d %s %s_scaled.img scale=%.3f clip=%i,%i edgenorm" \ % (avgs, avgs[:-4], self.scalefactor, 64, 64) avgs = avgs[:-4]+"_scaled.img" if start == "none": while os.path.isfile(avgs): apFile.removeStack(avgs) apParam.runCmd(emancmd, "EMAN") else: apDisplay.printColor("skipping stack scaling", "cyan") if self.imagicroot is not None and useEMAN1 is False: apIMAGIC.takeoverHeaders(avgs, npart, box) if self.imagicroot is not None: if self.params['prealign'] is True: if start == "none": avgs = apIMAGIC.prealignClassAverages(rundir, avgs) apIMAGIC.checkLogFileForErrors( os.path.join(rundir, "prealignClassAverages.log")) ### select between EMAN1 and IMAGIC for common lines if self.imagicroot is not None and useEMAN1 is False: commonlinesdir = os.path.join(rundir, "angular_reconstitution") else: commonlinesdir = os.path.join(rundir, "cross_common_lines") ### make links & calculate CCC matrix for 2D averages apDisplay.printColor("Calculating similarity matrix", "cyan") ccc_matrix = apCommonLines.calculate_ccc_matrix_2d(avgs) clsavgs = os.path.split(avgs)[1][:-4] if not os.path.isdir(commonlinesdir): os.mkdir(commonlinesdir) if os.path.islink(os.path.join(commonlinesdir, clsavgs+".hed")): os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".hed")) os.symlink(os.path.join(rundir, clsavgs+".hed"), os.path.join(commonlinesdir, clsavgs+".hed")) if os.path.islink(os.path.join(commonlinesdir, clsavgs+".img")): os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".img")) os.symlink(os.path.join(rundir, clsavgs+".img"), os.path.join(commonlinesdir, clsavgs+".img")) ################## create multiple 3d0s ##################### if start == "none": if self.imagicroot is not None and useEMAN1 is False: cmdlist = [] seqfile = open(os.path.join(rundir, "sequences_for_angular_reconstitution.dat"), "w") apDisplay.printColor("Running multiple raw volume calculations", "cyan") for i in range(nvol): sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart, ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws) apCommonLines.check_for_duplicates_in_sequence(sequence) seqfile.write(str(sequence)+"\n") ### create IMAGIC batch file for each model & append them to be threaded batchfile = apCommonLines.imagic_batch_file(sequence, i+1, avgs, sym, asqfilt, lmask, apix, box, anginc, keep_ordered, hamwin, lp, threes=threes, do_not_remove=False) proc = subprocess.Popen('chmod 755 '+batchfile, shell=True) proc.wait() cmdlist.append(batchfile) os.chdir(rundir) seqfile.close() apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10) ### check for errors after execution for i in range(nvol): apIMAGIC.checkLogFileForErrors( os.path.join(commonlinesdir, "3d"+str(i+1)+".log")) else: ### use EMAN1 cross-common lines cmdlist = [] seqfile = open(os.path.join(rundir, "sequences_for_cross_common_lines.dat"), "w") for i in range(nvol): sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart, ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws) apCommonLines.check_for_duplicates_in_sequence(sequence) seqfile.write(str(sequence)+"\n") vol_avgfile = os.path.join(commonlinesdir, "vol_%d_averages.hed" % (i+1)) vol_seqfile = os.path.join(commonlinesdir, "vol_%d_averages.txt" % (i+1)) vsf = open(vol_seqfile, "w") for j in range(ipv): vsf.write("%d\n" % sequence[j]) vsf.close() tmpdir = os.path.join(commonlinesdir, "tmp%d" % (i+1)) fullcmd = "proc2d %s %s list=%s ; " % (avgs, vol_avgfile, vol_seqfile) fullcmd+= "rm -rf %s ; mkdir %s ; cd %s ; " % (tmpdir, tmpdir, tmpdir) fullcmd+= "startAny %s sym=c1 proc=1 rounds=2 mask=%d ; " \ % (vol_avgfile, mrad/apix) fullcmd+= "mv threed.0a.mrc ../threed_%d.mrc ; " % (i+1) fullcmd+= "mv CCL.hed ../CCL_%d.hed ; " % (i+1) fullcmd+= "mv CCL.img ../CCL_%d.img ; " % (i+1) cmdlist.append(fullcmd) seqfile.close() apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10) for i in range(nvol): shutil.rmtree(os.path.join(commonlinesdir, "tmp%d" % (i+1))) else: apDisplay.printColor("skipping common lines volume calculations", "cyan") ############### move 3-D models volume directory ################ ### create volume directory volumedir = os.path.join(rundir, "volumes") if not os.path.isdir(volumedir): os.mkdir(volumedir) volumes = {} cmds = [] if start == "none": apDisplay.printColor("moving volumes for Xmipp 3-D Maximum Likelihood", "cyan") for i in range(nvol): if useEMAN1 is False: volume1 = os.path.join(commonlinesdir, "3d%d_ordered%d_filt.vol" % (i+1,i+1)) else: volume1 = os.path.join(commonlinesdir, "threed_%d.mrc" % (i+1)) volume2 = os.path.join(volumedir, "3d%d.vol" % (i+1)) if start == "none": if useEMAN1 is False: shutil.move(volume1, volume2) else: cmds.append("proc3d %s %s spidersingle" % (volume1, volume2)) volumes[(i+1)] = volume2 apThread.threadCommands(cmds, nproc=tnproc) #################### align 3-D models ################## if start == "3D_align" or start == "none": # if start != "3D_assess" and start != "3D_refine": ### run Maximum Likelihood 3-D alignment & align resulting volumes apDisplay.printColor("Running Xmipp maximum likelihood 3-D alignment", "cyan") vol_doc_file, alignref = apCommonLines.xmipp_max_like_3d_align( volumes, nproc, tnproc, nref) alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol) apDisplay.printColor("Aligning volumes based on 3-D ML parameters", "cyan") apCommonLines.align_volumes(alignparams, alignref, nvol, tnproc, apix) else: apDisplay.printColor("skipping 3D alignment", "cyan") vol_doc_file, alignref = apCommonLines.findAlignmentParams( os.path.join(rundir,"max_like_alignment"), nref) alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol) ##################### Principal Component Analysis ############## apDisplay.printColor("Calculating inter-volume similarity", "cyan") aligned_volumes = {} for i in range(nvol): aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.vol" % (i+1)) # aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.mrc" % (i+1)) if PCA is True: simfile, sim_matrix = apCommonLines.runPrincipalComponentAnalysis( aligned_volumes, nvol, neigens, box, apix, recalculate=recalc) else: simfile, sim_matrix = apCommonLines.calculate_ccc_matrix_3d( aligned_volumes, nvol) ################ 3-D affinity propagation ############# ### 3-D Affinity Propagation apDisplay.printColor("Averaging volumes with Affinity Propagation", "cyan") preffile = apCommonLines.set_preferences(sim_matrix, preftype, nvol) classes = apCommonLines.run_affinity_propagation(aligned_volumes, simfile, preffile, box, apix) ######### refine volumes using Xmipp projection matching ######### if start == "3D_align" or start == "3D_refine" or start == "none": # if start != "3D_assess": if not os.path.isdir("refinement"): os.mkdir("refinement") os.chdir("refinement") apXmipp.breakupStackIntoSingleFiles(ravgs) xmippcmd = "xmipp_normalize -i partlist.sel -method OldXmipp" apParam.runCmd(xmippcmd, "Xmipp") for i in classes.keys(): emancmd = "proc3d %s %s scale=%.3f clip=%d,%d,%d mask=%s spidersingle" \ % (os.path.join(rundir, "%d.mrc" % i), "%d.vol" % i, \ (1/self.scalefactor), rbox, rbox, rbox, (mrad / rapix)) apParam.runCmd(emancmd, "EMAN") apCommonLines.refine_volume(rundir, ravgs, i, mrad, irad, orad, rapix, nproc) emancmd = "proc3d %s %s apix=%.3f" \ % (os.path.join("refine_%d" % i, "3d%d_refined.vol" % i), \ os.path.join(rundir, "%d_r.mrc" % i), rapix) apParam.runCmd(emancmd, "EMAN") os.chdir(rundir) else: apDisplay.printColor("skipping 3D refinement", "cyan") ################# model evaluation ################ if start == "3D_align" or start == "3D_refine" or start == "3D_assess" or start == "none": # if start == "none" or start=="3D_assess": ### final model assessment try: euler_array = apCommonLines.getEulerValuesForModels(alignparams, rundir, nvol, npart, threes=threes) except: euler_array = None ejassess = False if refine is True: apCommonLines.assess_3Dclass_quality2(rundir, aligned_volumes, sim_matrix, classes, euler_array, box, rbox, rapix, ravgs, psym, npart, ejassess=ejassess) else: apCommonLines.assess_3Dclass_quality(rundir, aligned_volumes, sim_matrix, classes, euler_array, box, apix, avgs, psym, npart, ejassess=ejassess) apCommonLines.combineMetrics("final_model_stats.dat", "final_model_stats_sorted_by_Rcrit.dat", **{"CCPR":(1,1)}) ################## upload & cleanup ################## ### upload to database, if specified self.upload() ### make chimera snapshots if refine is True: for i in classes.keys(): if self.params['mass'] is not None: apChimera.filterAndChimera( os.path.join(self.params['rundir'], "%d_r.mrc" % i), res=self.params['3d_lpfilt'], apix=self.params['refineapix'], box=self.params['refineboxsize'], chimtype="snapshot", contour=2, zoom=1, sym="c1", color="gold", mass=self.params['mass'] ) ### cleanup snapshots = glob.glob("*.png") mtlfiles = glob.glob("*.mtl") objfiles = glob.glob("*.obj") pyfiles = glob.glob("*mrc.py") for file in mtlfiles: os.remove(file) for file in objfiles: os.remove(file) for file in pyfiles: os.remove(file) if not os.path.isdir("snapshots"): os.mkdir("snapshots") for s in snapshots: shutil.move(s, os.path.join("snapshots", s))
return parser.parse_args() def checkConflicts(options, args): ### metrics with values if options.infile == None: errmsg = "specify the input filename with quality metrics and associated values " errmsg+= "e.g. --infile=final_model_stats.dat" apDisplay.printError(errmsg) if options.outfile == None: errmsg = "specify the output filename with quality metrics and associated values " errmsg+= "e.g. --outfile=final_model_stats_sorted_by_Rcrit.dat" apDisplay.printError(errmsg) if options.metrics == None: errmsg = "please specify the desired metric(s) to combine, along with appropriate " errmsg+= "weight and sign, for example: --metrics='CCPR,1,1 EJ,0.5,-1 FSC,0.1,-1'" apDisplay.printError(errmsg) return ########################## if __name__ == "__main__": options, args = setupParserOptions() checkConflicts(options, args) metrics = options.metrics.split() kwargs = {} for metric in metrics: m, weight, sign = metric.split(",") kwargs[m] = (float(weight),int(sign)) apCommonLines.combineMetrics(options.infile, options.outfile, **kwargs)