def correctCTFToDocFile(self, indocfile):
		apDisplay.printMsg("Correcting CTF applied to particles")

		inf = open(indocfile, 'r')
		cmdlist = []
		partnum = 0
		for line in inf:
			### get filename
			partnum += 1
			sline = line.strip()
			if not sline:
				continue
			filename = sline.split()[0]

			### correct CTF using ACE2
			ace2cmd = (self.ace2correct+" -img %s -kv %d -cs %.3f -apix %.3f -df %.9f,%.9f,%.3f -wiener 0.1 -ampc %.3f"
				%(filename, self.params['kv'], self.params['cs'], self.params['apix'],
				self.deflist1c[partnum-1], self.deflist2c[partnum-1], self.params['astigmatism'], self.params['amp']))
			cmdlist.append(ace2cmd)

		numpart = partnum
		inf.close()

		### thread the commands
		t0 = time.time()
		apThread.threadCommands(cmdlist)
		timeper = (time.time()-t0)/float(numpart)
		apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green")
		apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green")

		### check for the files, write to doc file
		inf = open(indocfile, 'r')
		outdocfile = os.path.splitext(indocfile)[0]+".correct.lst"
		outf = open(outdocfile, 'w')
		for line in inf:
			### get filename
			filename = line.strip().split()[0]
			newfile = filename+".corrected.mrc"
			if not os.path.isfile(newfile):
				apDisplay.printError("Ace 2 failed")
			outf.write(newfile+"\t1\n")
		inf.close()
		outf.close()

		return outdocfile
    def start(self):
        ### get volume files
        volumefiles = glob.glob(self.params['volumes'])
        if not volumefiles:
            apDisplay.printError("Could not find volumes, %s" %
                                 (self.params['volumes']))

        ### make list of all alignments to run
        cmdlist = []
        alignfiles = []
        for volfile in volumefiles:
            alignfile = "align-" + os.path.basename(volfile)
            alignfiles.append(alignfile)
            emancmd = "align3d %s %s %s " % (self.params['reference'], volfile,
                                             alignfile)
            if self.params['slow'] is True:
                emancmd += "slow "
            if self.params['noshrink'] is True:
                emancmd += "noshrink "
            #print emancmd
            cmdlist.append(emancmd)

        ### run several alignment commands in parallel
        apThread.threadCommands(cmdlist)

        ### average volumes together
        ref = mrc.read(self.params['reference'])
        average = numpy.zeros(ref.shape, dtype=numpy.float32)
        del ref
        count = 0
        for alignfile in alignfiles:
            if not os.path.isfile(alignfile):
                apDisplay.printWarning("aligned volume not found: %s" %
                                       (alignfile))
            aligned = mrc.read(alignfile)
            count += 1
            ### this assume all aligned volume have same box size
            average += aligned
            del aligned

        ### save average
        average /= count
        avgfile = os.path.abspath(self.params['average'])
        mrc.write(average, avgfile)
        apDisplay.printMsg("Wrote average file: " + avgfile)
    def singleNodeRun(self, iternum):
        """
                single node run
                """
        ### single node run
        apDisplay.printMsg("Single node run, iteration %d" % (iternum))

        ### create individual processor jobs
        stackbase = os.path.splitext(os.path.basename(self.refinestackfile))[0]
        self.stackfile = "../../%s" % (stackbase)
        procjobfiles = self.createMultipleJobs(iternum)

        ### convert job files to commands
        procjobcmds = []
        for procjobfile in procjobfiles:
            procjobcmd = "sh " + procjobfile
            procjobcmds.append(procjobcmd)

        ### run individual processor jobs
        t0 = time.time()
        apThread.threadCommands(procjobcmds,
                                nproc=self.params['nproc'],
                                pausetime=30.0)
        apDisplay.printColor(
            "Refinement complete in %s" %
            (apDisplay.timeString(time.time() - t0)), "green")

        ### create combine processor jobs
        stackbase = os.path.splitext(os.path.basename(self.reconstackfile))[0]
        self.stackfile = "../%s" % (stackbase)
        combinejobfile = self.combineMultipleJobs(iternum)

        ### run combine processor jobs
        t0 = time.time()
        proc = subprocess.Popen("sh " + combinejobfile, shell=True)
        proc.wait()
        apDisplay.printColor(
            "Volume complete in %s" % (apDisplay.timeString(time.time() - t0)),
            "green")

        if apFile.fileSize('iter%03d.img' % (iternum)) < 100:
            apDisplay.printError("Failed to generate volume for iter %d" %
                                 (iternum))

        return
	def correctCTFToDocFile(self, indocfile):
		apDisplay.printMsg("Correcting CTF applied to particles")

		inf = open(indocfile, 'r')
		cmdlist = []
		partnum = 0
		for line in inf:
			### get filename
			partnum += 1
			sline = line.strip()
			if not sline:
				continue
			filename = sline.split()[0]

			### correct CTF using ACE2
			ace2cmd = (self.ace2correct+" -img %s -kv %d -cs %.3f -apix %.3f -df %.9f,%.9f,%.3f -wiener 0.1 -ampc %.3f"
				%(filename, self.params['kv'], self.params['cs'], self.params['apix'],
				self.deflist1c[partnum-1], self.deflist2c[partnum-1], self.params['astigmatism'], self.params['amp']))
			cmdlist.append(ace2cmd)

		numpart = partnum
		inf.close()

		### thread the commands
		t0 = time.time()
		apThread.threadCommands(cmdlist)
		timeper = (time.time()-t0)/float(numpart)
		apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green")
		apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green")

		### check for the files, write to doc file
		inf = open(indocfile, 'r')
		outdocfile = os.path.splitext(indocfile)[0]+".correct.lst"
		outf = open(outdocfile, 'w')
		for line in inf:
			### get filename
			filename = line.strip().split()[0]
			newfile = filename+".corrected.mrc"
			if not os.path.isfile(newfile):
				apDisplay.printError("Ace 2 failed")
			outf.write(newfile+"\t1\n")
		inf.close()
		outf.close()

		return outdocfile
	def start(self):
		### get volume files
		volumefiles = glob.glob(self.params['volumes'])
		if not volumefiles:
			apDisplay.printError("Could not find volumes, %s"%(self.params['volumes']))

		### make list of all alignments to run
		cmdlist = []
		alignfiles = []
		for volfile in volumefiles:
			alignfile = "align-"+os.path.basename(volfile)
			alignfiles.append(alignfile)
			emancmd = "align3d %s %s %s "%(self.params['reference'], volfile, alignfile)
			if self.params['slow'] is True:
				emancmd += "slow "
			if self.params['noshrink'] is True:
				emancmd += "noshrink "
			#print emancmd
			cmdlist.append(emancmd)

		### run several alignment commands in parallel
		apThread.threadCommands(cmdlist)

		### average volumes together
		ref = mrc.read(self.params['reference'])
		average = numpy.zeros(ref.shape, dtype=numpy.float32)
		del ref
		count = 0
		for alignfile in alignfiles:
			if not os.path.isfile(alignfile):
				apDisplay.printWarning("aligned volume not found: %s"%(alignfile))
			aligned = mrc.read(alignfile)
			count += 1
			### this assume all aligned volume have same box size
			average += aligned
			del aligned

		### save average
		average /= count
		avgfile = os.path.abspath(self.params['average'])
		mrc.write(average, avgfile)
		apDisplay.printMsg("Wrote average file: "+avgfile)
	def singleNodeRun(self, iternum):
		"""
		single node run
		"""
		### single node run
		apDisplay.printMsg("Single node run, iteration %d"%(iternum))

		### create individual processor jobs
		stackbase = os.path.splitext(os.path.basename(self.refinestackfile))[0]
		self.stackfile = "../../%s"%(stackbase)
		procjobfiles = self.createMultipleJobs(iternum)

		### convert job files to commands
		procjobcmds = []
		for procjobfile in procjobfiles:
			procjobcmd = "sh "+procjobfile
			procjobcmds.append(procjobcmd)

		### run individual processor jobs
		t0 = time.time()
		apThread.threadCommands(procjobcmds, nproc=self.params['nproc'], pausetime=30.0)
		apDisplay.printColor("Refinement complete in %s"%(apDisplay.timeString(time.time()-t0)), "green")

		### create combine processor jobs
		stackbase = os.path.splitext(os.path.basename(self.reconstackfile))[0]
		self.stackfile = "../%s"%(stackbase)
		combinejobfile = self.combineMultipleJobs(iternum)


		### run combine processor jobs
		t0 = time.time()
		proc = subprocess.Popen("sh "+combinejobfile, shell=True)
		proc.wait()
		apDisplay.printColor("Volume complete in %s"%(apDisplay.timeString(time.time()-t0)), "green")

		if apFile.fileSize('iter%03d.img'%(iternum)) < 100:
			apDisplay.printError("Failed to generate volume for iter %d"%(iternum))

		return
Example #7
0
	def start(self):
		''' 
		OptiMod script 
		'''
		
		###############     define short-hand parameter names       ############
			
		refine = True
		rundir = self.params['rundir']
		box = self.params['boxsize']
		rbox = self.params['refineboxsize']
		apix = self.params['apix']
		rapix = self.params['refineapix']
		npart = self.params['numpart']
		nvol = self.params['num_volumes']
#		sym = self.params['sym']
		sym = "c1"
		psym = self.params['presumed_sym']
		first = self.params['firstimage']
		nws = self.params['non_weighted_sequence']
		threes = self.params['threes']
		lmask = self.params['linmask']
		asqfilt = self.params['asqfilt']
		anginc = self.params['ang_inc']
		keep_ordered = self.params['keep_ordered_num']
		hamwin = self.params['ham_win']
		useEMAN1 = self.params['useEMAN1']
		ipv = self.params['images_per_volume']
		lp = self.params['3d_lpfilt']
		nref = self.params['nref']
		PCA = self.params['PCA']
		neigens = self.params['numeigens']
		recalc = self.params['recalculate']
		preftype = self.params['preftype']
		mrad = self.params['mask_radius']
		irad = self.params['inner_radius']
		orad = self.params['outer_radius']
		tnproc = self.params['threadnproc']
		nproc = self.params['nproc']
		oldavgs = self.params['oldavgs']
		avgs = os.path.join(self.params['rundir'], self.params['avgs'])
		start = self.params['start_at']
				
		#############            copy to working directory          ############
		
		if not os.path.isfile(os.path.join(avgs)[:-4]+".hed"):
			shutil.copyfile(os.path.join(oldavgs[:-4]+".hed"), avgs[:-4]+".hed")				
		if not os.path.isfile(os.path.join(avgs)[:-4]+".img"):
			shutil.copyfile(os.path.join(oldavgs[:-4]+".img"), avgs[:-4]+".img")				
		if self.params['ravgs'] is not None:
			ravgs = os.path.join(self.params['rundir'], self.params['ravgs'])
			if not os.path.isfile(self.params['ravgs'][:-4]+".hed"):
				shutil.copy(self.params['oldravgs'][:-4]+".hed", 
					os.path.join(self.params['ravgs'][:-4]+".hed"))
			if not os.path.isfile(self.params['ravgs'][:-4]+".img"):
				shutil.copy(self.params['oldravgs'][:-4]+".img", 
					os.path.join(self.params['ravgs'][:-4]+".img"))			
			### Euler jumper assessment does not make sense when refine images are different
			ejassess = False 
		else:
			ravgs = avgs
			ejassess = True
						
		###########    scale & prealign class averages, if specified   #########
		
		if self.params['scale'] is True:
			emancmd = "proc2d %s %s_scaled.img scale=%.3f clip=%i,%i edgenorm" \
				% (avgs, avgs[:-4], self.scalefactor, 64, 64)
			avgs = avgs[:-4]+"_scaled.img"
			if start == "none":
				while os.path.isfile(avgs):
					apFile.removeStack(avgs)
				apParam.runCmd(emancmd, "EMAN")
			else:
				apDisplay.printColor("skipping stack scaling", "cyan")
		if self.imagicroot is not None and useEMAN1 is False:
			apIMAGIC.takeoverHeaders(avgs, npart, box)
		
		if self.imagicroot is not None:
			if self.params['prealign'] is True:
				if start == "none":
					avgs = apIMAGIC.prealignClassAverages(rundir, avgs)
					apIMAGIC.checkLogFileForErrors(
						os.path.join(rundir, "prealignClassAverages.log"))

		### select between EMAN1 and IMAGIC for common lines	
		if self.imagicroot is not None and useEMAN1 is False:
			commonlinesdir = os.path.join(rundir, "angular_reconstitution")
		else:
			commonlinesdir = os.path.join(rundir, "cross_common_lines")

		### make links & calculate CCC matrix for 2D averages
		apDisplay.printColor("Calculating similarity matrix", "cyan")
		ccc_matrix = apCommonLines.calculate_ccc_matrix_2d(avgs)
		clsavgs = os.path.split(avgs)[1][:-4]
		if not os.path.isdir(commonlinesdir):
			os.mkdir(commonlinesdir)
		if os.path.islink(os.path.join(commonlinesdir, clsavgs+".hed")):
			os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".hed"))
		os.symlink(os.path.join(rundir, clsavgs+".hed"), 
			os.path.join(commonlinesdir, clsavgs+".hed"))
		if os.path.islink(os.path.join(commonlinesdir, clsavgs+".img")):
			os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".img"))
		os.symlink(os.path.join(rundir, clsavgs+".img"), 
			os.path.join(commonlinesdir, clsavgs+".img"))

		##################      create multiple 3d0s       #####################
		
		if start == "none":
			if self.imagicroot is not None and useEMAN1 is False:
				cmdlist = []
				seqfile = open(os.path.join(rundir, 
					"sequences_for_angular_reconstitution.dat"), "w")
				apDisplay.printColor("Running multiple raw volume calculations", "cyan")
				for i in range(nvol):
					sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart, 
						ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
					apCommonLines.check_for_duplicates_in_sequence(sequence)
					seqfile.write(str(sequence)+"\n")
					### create IMAGIC batch file for each model & append them to be threaded
					batchfile = apCommonLines.imagic_batch_file(sequence, i+1, avgs, sym, 
						asqfilt, lmask, apix, box, anginc, keep_ordered, hamwin, lp, 
						threes=threes, do_not_remove=False)
					proc = subprocess.Popen('chmod 755 '+batchfile, shell=True)
					proc.wait()
					cmdlist.append(batchfile)
					os.chdir(rundir)
				seqfile.close()
				apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)
		
				### check for errors after execution
				for i in range(nvol):
					apIMAGIC.checkLogFileForErrors(
						os.path.join(commonlinesdir, "3d"+str(i+1)+".log"))
			else:
				### use EMAN1 cross-common lines
				cmdlist = []
				seqfile = open(os.path.join(rundir, "sequences_for_cross_common_lines.dat"), "w")
				for i in range(nvol):
					sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart, 
						ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
					apCommonLines.check_for_duplicates_in_sequence(sequence)
					seqfile.write(str(sequence)+"\n")
					vol_avgfile = os.path.join(commonlinesdir, "vol_%d_averages.hed" % (i+1))
					vol_seqfile = os.path.join(commonlinesdir, "vol_%d_averages.txt" % (i+1))
					vsf = open(vol_seqfile, "w")
					for j in range(ipv):
						vsf.write("%d\n" % sequence[j])
					vsf.close()
					tmpdir = os.path.join(commonlinesdir, "tmp%d" % (i+1))				
					fullcmd = "proc2d %s %s list=%s ; " % (avgs, vol_avgfile, vol_seqfile)
					fullcmd+= "rm -rf %s ; mkdir %s ; cd %s ; " % (tmpdir, tmpdir, tmpdir)
					fullcmd+= "startAny %s sym=c1 proc=1 rounds=2 mask=%d ; " \
						% (vol_avgfile, mrad/apix)
					fullcmd+= "mv threed.0a.mrc ../threed_%d.mrc ; " % (i+1)
					fullcmd+= "mv CCL.hed ../CCL_%d.hed ; " % (i+1)
					fullcmd+= "mv CCL.img ../CCL_%d.img ; " % (i+1)
					cmdlist.append(fullcmd)
				seqfile.close()
				apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)	
				for i in range(nvol):
					shutil.rmtree(os.path.join(commonlinesdir,	"tmp%d" % (i+1)))		
		else:
			apDisplay.printColor("skipping common lines volume calculations", "cyan")
			
		###############     move 3-D models volume directory    ################
		
		### create volume directory
		volumedir = os.path.join(rundir, "volumes")
		if not os.path.isdir(volumedir):
			os.mkdir(volumedir)
		volumes = {}
		cmds = []
		if start == "none":
			apDisplay.printColor("moving volumes for Xmipp 3-D Maximum Likelihood", "cyan")
		for i in range(nvol):
			if useEMAN1 is False:
				volume1 = os.path.join(commonlinesdir, "3d%d_ordered%d_filt.vol" % (i+1,i+1))
			else:
				volume1 = os.path.join(commonlinesdir, "threed_%d.mrc" % (i+1))
			volume2 = os.path.join(volumedir, "3d%d.vol" % (i+1))
			if start == "none":
				if useEMAN1 is False:
					shutil.move(volume1, volume2)
				else:
					cmds.append("proc3d %s %s spidersingle" % (volume1, volume2))
			volumes[(i+1)] = volume2		
		apThread.threadCommands(cmds, nproc=tnproc)
		
		####################          align 3-D models        ##################

		if start == "3D_align" or start == "none":
#		if start != "3D_assess" and start != "3D_refine":		
			### run Maximum Likelihood 3-D alignment & align resulting volumes
			apDisplay.printColor("Running Xmipp maximum likelihood 3-D alignment", "cyan")
			vol_doc_file, alignref = apCommonLines.xmipp_max_like_3d_align(
				volumes, nproc, tnproc, nref)
			alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)
			apDisplay.printColor("Aligning volumes based on 3-D ML parameters", "cyan")
			apCommonLines.align_volumes(alignparams, alignref, nvol, tnproc, apix)
		else:
			apDisplay.printColor("skipping 3D alignment", "cyan")
			vol_doc_file, alignref = apCommonLines.findAlignmentParams(
				os.path.join(rundir,"max_like_alignment"), nref)
			alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)

		#####################      Principal Component Analysis   ##############
		
		apDisplay.printColor("Calculating inter-volume similarity", "cyan")
		aligned_volumes = {}
		for i in range(nvol):
			aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.vol" % (i+1))
#			aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.mrc" % (i+1))
		if PCA is True:
			simfile, sim_matrix = apCommonLines.runPrincipalComponentAnalysis(
				aligned_volumes, nvol, neigens, box, apix, recalculate=recalc)
		else:
			simfile, sim_matrix = apCommonLines.calculate_ccc_matrix_3d(
				aligned_volumes, nvol)	

		################            3-D affinity propagation       #############

		### 3-D Affinity Propagation
		apDisplay.printColor("Averaging volumes with Affinity Propagation", "cyan")
		preffile = apCommonLines.set_preferences(sim_matrix, preftype, nvol)
		classes = apCommonLines.run_affinity_propagation(aligned_volumes, simfile, 
			preffile, box, apix)

		#########    refine volumes using Xmipp projection matching    #########

		if start == "3D_align" or start == "3D_refine" or start == "none":
#		if start != "3D_assess":
			if not os.path.isdir("refinement"):
				os.mkdir("refinement")
			os.chdir("refinement")
			apXmipp.breakupStackIntoSingleFiles(ravgs)
			xmippcmd = "xmipp_normalize -i partlist.sel -method OldXmipp"
			apParam.runCmd(xmippcmd, "Xmipp")
			for i in classes.keys():
				emancmd = "proc3d %s %s scale=%.3f clip=%d,%d,%d mask=%s spidersingle" \
					% (os.path.join(rundir, "%d.mrc" % i), "%d.vol" % i, \
						(1/self.scalefactor), rbox, rbox, rbox, (mrad / rapix))
				apParam.runCmd(emancmd, "EMAN")
				apCommonLines.refine_volume(rundir, ravgs, i, mrad, irad, orad, rapix, nproc)
				emancmd = "proc3d %s %s apix=%.3f" \
					% (os.path.join("refine_%d" % i, "3d%d_refined.vol" % i), \
						os.path.join(rundir, "%d_r.mrc" % i), rapix)
				apParam.runCmd(emancmd, "EMAN")
			os.chdir(rundir)
		else:
			apDisplay.printColor("skipping 3D refinement", "cyan")			
		
		#################            model evaluation           ################
		
		if start == "3D_align" or start == "3D_refine" or start == "3D_assess" or start == "none":
#		if start == "none" or start=="3D_assess":
			### final model assessment
			try:
				euler_array = apCommonLines.getEulerValuesForModels(alignparams, 
					rundir, nvol, npart, threes=threes)
			except:
				euler_array = None
				ejassess = False
			if refine is True:
				apCommonLines.assess_3Dclass_quality2(rundir, aligned_volumes, sim_matrix, 
					classes, euler_array, box, rbox, rapix, ravgs, psym, npart, ejassess=ejassess)
			else:
				apCommonLines.assess_3Dclass_quality(rundir, aligned_volumes, sim_matrix, 
					classes, euler_array, box, apix, avgs, psym, npart, ejassess=ejassess)
			apCommonLines.combineMetrics("final_model_stats.dat", 
				"final_model_stats_sorted_by_Rcrit.dat", **{"CCPR":(1,1)})


		##################          upload & cleanup          ##################

		### upload to database, if specified
		self.upload()
		
		### make chimera snapshots
		if refine is True:
			for i in classes.keys():
				if self.params['mass'] is not None:
					apChimera.filterAndChimera(
						os.path.join(self.params['rundir'], "%d_r.mrc" % i),
						res=self.params['3d_lpfilt'],
						apix=self.params['refineapix'],
						box=self.params['refineboxsize'],
						chimtype="snapshot",
						contour=2,
						zoom=1,
						sym="c1",
						color="gold",
						mass=self.params['mass']
						)
	
		### cleanup
		snapshots = glob.glob("*.png")
		mtlfiles = glob.glob("*.mtl")
		objfiles = glob.glob("*.obj")
		pyfiles = glob.glob("*mrc.py")
		for file in mtlfiles:
			os.remove(file)
		for file in objfiles:
			os.remove(file)
		for file in pyfiles:
			os.remove(file)
		if not os.path.isdir("snapshots"):
			os.mkdir("snapshots")
		for s in snapshots:
			shutil.move(s, os.path.join("snapshots", s))
 def runMultipleFreeHandTest(self, commandlist):
     apThread.threadCommands(commandlist,
                             nproc=self.params['nproc'],
                             pausetime=1.0)
 def runMultipleFreeHandTest(self, commandlist):
     apThread.threadCommands(commandlist, nproc=self.params["nproc"], pausetime=1.0)