def createChimeraVolumeSnapshot(self, volume, iteration, reference_number=1):
        """ create chimera snapshot images of volume """

        if self.params["snapfilter"]:
            resolution = self.params["snapfilter"]
        else:
            newfscfile = os.path.join(
                self.resultspath,
                "recon_%s_it%.3d_vol%.3d.fsc" % (self.params["timestamp"], iteration, reference_number),
            )
            try:
                resolution = apRecon.getResolutionFromGenericFSCFile(
                    newfscfile, self.runparams["boxsize"], self.runparams["apix"]
                )
            except:
                apDisplay.printWarning("Failed to get resolution from generic FSC file: " + newfscfile)
                resolution = 30
        apDisplay.printWarning("Running Chimera Snapshot with resolution: %d " % resolution)
        apChimera.filterAndChimera(
            volume,
            resolution,
            self.runparams["apix"],
            self.runparams["boxsize"],
            "snapshot",
            self.params["contour"],
            self.params["zoom"],
            sym="c1",
            mass=self.params["mass"],
        )
	def checkConflicts(self):
		if self.params['stackid'] is None:
			apDisplay.printError("stack id was not defined")
		if self.params['rad'] is None:
			apDisplay.printError("no particle radius set")
		if self.params['modelid'] is None:
			apDisplay.printError("model id was not defined")
		if self.params['lastring'] is None:
			apDisplay.printError("a last ring radius was not provided")
		if self.params['runname'] is None:
			apDisplay.printError("run name was not defined")
		if self.params['incr'] is None:
			apDisplay.printError("angular increments are not specified")
		stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
		stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
		numstackp = apFile.numImagesInStack(stackfile)
		if self.params['numpart'] > numstackp:
			apDisplay.printError("trying to use more particles "+str(numstackp)
				+" than available "+str(numstackp))
		elif self.params['numpart'] == 0:
			apDisplay.printWarning("using all "+str(numstackp)+" particles")
			self.params['numpart'] = numstackp
		boxsize = apStack.getStackBoxsize(self.params['stackid'])
		if self.params['lastring'] > boxsize/2-2:
			apDisplay.printError("last ring radius is too big for boxsize "
				+str(self.params['lastring'])+" > "+str(boxsize/2-2))
		if self.params['lastring']+self.params['xysearch'] > boxsize/2-2:
			apDisplay.printError("last ring plus xysearch radius is too big for boxsize "
				+str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2))
		if (self.params['xysearch'] % self.params['xystep']) > 0:
			apDisplay.printError("translational search (xy-search) must be divisible by search step (xy-step)")
        def _initializeDoneDict(self):
                """
                reads or creates a done dictionary
                """
                self.donedictfile = os.path.join(self.params['rundir'] , self.functionname+".donedict")
                if os.path.isfile(self.donedictfile) and self.params['continue'] == True:
                        ### unpickle previously done dictionary
                        apDisplay.printMsg("Reading old done dictionary: "+os.path.basename(self.donedictfile))
                        f = open(self.donedictfile,'r')
                        self.donedict = cPickle.load(f)
                        f.close()
                        if not 'commit' in self.donedict or self.donedict['commit'] == self.params['commit']:
                                ### all is well
                                apDisplay.printMsg("Found "+str(len(self.donedict))+" done dictionary entries")
                                return
                        elif self.donedict['commit'] is True and self.params['commit'] is not True:
                                ### die
                                apDisplay.printError("Commit flag was enabled and is now disabled, create a new runname")
                        else:
                                ### set up fresh dictionary
                                apDisplay.printWarning("'--commit' flag was changed, creating new done dictionary")

                ### set up fresh dictionary
                self.donedict = {}
                self.donedict['commit'] = self.params['commit']
                apDisplay.printMsg("Creating new done dictionary: "+os.path.basename(self.donedictfile))

                ### write donedict to file
                f = open(self.donedictfile, 'w', 0666)
                cPickle.dump(self.donedict, f)
                f.close()
                return
def runChimeraScript(chimscript, xvfb=False):
        if not chimscript or not os.path.isfile(chimscript):
                print chimscript
                apDisplay.printError("Could not find file: apChimSnapshot.py")
        #apDisplay.printColor("Trying to use chimera for model imaging","cyan")
        if xvfb is True:
                port = apParam.resetVirtualFrameBuffer()
                time.sleep(1)
        if 'CHIMERA' in os.environ and os.path.isdir(os.environ['CHIMERA']):
                chimpath = os.environ['CHIMERA']
                os.environ['CHIMERA'] = chimpath
                os.environ['CHIMERAPATH'] = os.path.join(chimpath,"share")
                os.environ['LD_LIBRARY_PATH'] = os.path.join(chimpath,"lib")+":"+os.environ['LD_LIBRARY_PATH']
                chimexe = os.path.join(chimpath,"bin/chimera")
                if not os.path.isfile(chimexe):
                        apDisplay.printWarning("Could not find chimera at: "+chimexe)
        else:
                chimpath = None
                chimexe = "chimera"
                #apDisplay.printWarning("'CHIMERA' environmental variable is unset")
        rendercmd = (chimexe+" --debug python:"+chimscript)
        logf = open("chimeraRun.log", "a")
        apDisplay.printColor("running Chimera:\n "+rendercmd, "cyan")
        if xvfb is True:
                print "import -verbose -display :%d -window root screencapture.png"%(port)
        proc = subprocess.Popen(rendercmd, shell=True, stdout=logf, stderr=logf)
        proc.wait()
        logf.close()
        if xvfb is True:
                apParam.killVirtualFrameBuffer(port)
        return
	def start(self):
		"""
		this is the main component of the script
		where all the processing is done
		"""
		### initialize some variables
		self.runq = None
		self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
		apDisplay.printMsg("Pixel size: %.5f"%(self.apix))
		self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
		apDisplay.printMsg("Box size: %d"%(self.boxsize))

		self.checkResults()
		self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
		self.numiter = self.getNumberOfIterations()
		for i in range(self.numiter):
			iternum = i+1
			apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green")
			self.uploadIteration(iternum)

		reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir'])
		if reconrunid:
			apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
			eulerjump = apEulerJump.ApEulerJump()
			eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid'])
			apRecon.setGoodBadParticlesFromReconId(reconrunid)
		else:
			apDisplay.printWarning("Could not find recon run id")
	def start(self):
		self.setFileName()

		scale =  float(self.params['oldapix'])/self.params['newapix']

		mrcname = os.path.join(self.params['rundir'], self.params['name']+".mrc")
		origmodel = self.params['file']
		if os.path.isfile(mrcname):
			apDisplay.printError("File exists")

		if (abs(self.params['oldapix'] - self.params['newapix']) > 1.0e-2 or
			abs(self.params['oldbox'] - self.params['newbox']) > 1.0e-1):
			### rescale old model to a new size
			apDisplay.printWarning("rescaling original model to a new size")
			scale = float(self.params['oldapix'])/self.params['newapix']
			apDisplay.printMsg("rescaling model by scale factor of %.4f"%(scale))
			apVolume.rescaleVolume(origmodel, mrcname,
				self.params['oldapix'], self.params['newapix'], self.params['newbox'])
		else:
			### simple upload, just copy file to models folder
			apDisplay.printMsg("copying original model to a new location: "+mrcname)
			shutil.copyfile(origmodel, mrcname)

		if self.params['viper2eman'] is True:
			apVolume.viper2eman(mrcname, mrcname, apix=self.params['newapix'])

		### render chimera images of model
		contour = self.params['contour']
		if self.params['mass'] is not None:
			apChimera.setVolumeMass(mrcname, self.params['newapix'], self.params['mass'])
			contour = 1.0
		apChimera.renderSnapshots(mrcname, contour=contour,
			zoom=self.params['zoom'], sym=self.params['symdata']['eman_name'])

		self.insertModel(mrcname)
        def fixAmpContrast(self, defocus, raddata, PSDarray, lowerbound, upperbound):
                if self.ctfvalues['amplitude_contrast'] > 0 and self.ctfvalues['amplitude_contrast'] < self.maxAmpCon:
                        return defocus
                bad = True
                newdefocus = defocus
                count = 0
                while count < 20:
                        count += 1
                        amplitudecontrast = sinefit.refineAmplitudeContrast(raddata[lowerbound:upperbound]*1e10, newdefocus, 
                                PSDarray[lowerbound:upperbound], self.ctfvalues['cs'], self.wavelength, msg=self.debug)
                        if amplitudecontrast is None:
                                apDisplay.printWarning("FAILED to fix amplitude contrast")
                                return defocus
                        elif amplitudecontrast < 0:
                                apDisplay.printColor("Amp Cont: %.3f too small, decrease defocus %.3e"%
                                        (amplitudecontrast, newdefocus), "blue")
                                scaleFactor = 0.999 - abs(amplitudecontrast)/5.
                                newdefocus = newdefocus*scaleFactor
                        elif amplitudecontrast > self.maxAmpCon:
                                apDisplay.printColor("Amp Cont: %.3f too large, increase defocus %.3e"%
                                        (amplitudecontrast, newdefocus), "cyan")
                                scaleFactor = 1.001 + abs(amplitudecontrast - self.maxAmpCon)/5.
                                newdefocus = newdefocus*scaleFactor
                        else:
                                apDisplay.printColor("Amp Cont: %.3f in range!!!  defocus %.3e"%
                                        (amplitudecontrast, newdefocus), "green")
                                avgres = self.getResolution(newdefocus, raddata, PSDarray, lowerbound)
                                self.ctfvalues['amplitude_contrast'] = amplitudecontrast
                                return newdefocus

                        avgres = self.getResolution(newdefocus, raddata, PSDarray, lowerbound)
                        if avgres < self.bestres:
                                defocus = newdefocus
                apDisplay.printWarning("FAILED to fix amplitude contrast")
                return defocus
	def getGenerationFiles(self):
		if self.files is None:
			files = glob.glob("class_averages_generation_*.hdf")
			if len(files) == 0:
				apDisplay.printWarning("Could not find class_averages_generation_*.hdf files.")
			self.files = sorted(files, key=lambda a: self.generationFromFile(a))
		return self.files
    def insertFSCData(self, fscfile, refineIterData):
        """ 
                current functions read the inverse pixel number and correlation value. This function, therefore, converts
                generic FSC parameters into pixel / correlation format
                """

        if not os.path.isfile(fscfile):
            apDisplay.printWarning("Could not open FSC file: " + fscfile)

        ### skip commented out lines and read FSC data
        f = open(fscfile, "r")
        fscfileinfo = f.readlines()
        f.close()
        for i, info in enumerate(fscfileinfo):
            if info[0] == "#":
                pass
            else:
                fscfileinfo = fscfileinfo[i:]
                break

        ### insert FSC data into database with the convention: pixel number, FSC
        numinserts = 0
        for j, info in enumerate(fscfileinfo):
            fscq = appiondata.ApFSCData()
            fscq["refineIter"] = refineIterData
            fscq["pix"] = j + 1
            fscq["value"] = float(info.split()[1])
            numinserts += 1
            if self.params["commit"] is True:
                fscq.insert()
        if self.params["commit"] is True:
            apDisplay.printMsg("inserted " + str(numinserts) + " rows of FSC data into database")

        return
def setVolumeMass(volumefile, apix=1.0, mass=1.0, rna=0.0):
        """
        set the contour of 1.0 to the desired mass (in kDa) of the
        macromolecule based on its density
        
        use RNA to set the percentage of RNA in the structure
        """
        if isValidVolume(volumefile) is False:
                apDisplay.printError("Volume file is not valid")

        procbin = apParam.getExecPath("proc2d")
        emandir = os.path.dirname(procbin)
        volumebin = os.path.join(emandir, "volume")
        if not os.path.isfile(volumebin):
                apDisplay.printWarning("failed to find volume program")
                return False
        command = "%s %s %.3f set=%.3f"%(       
                volumebin, volumefile, apix, mass
        )
        t0 = time.time()
        proc = subprocess.Popen(command, shell=True)
        proc.wait()
        if time.time()-t0 < 0.01:
                apDisplay.printWarning("failed to scale by mass in "+apDisplay.timeString(time.time()-t0))
                return False
        apDisplay.printMsg("finished scaling by mass in "+apDisplay.timeString(time.time()-t0))
        return True
Esempio n. 11
0
	def checkSelOrDocFileRootDirectory(self, sel_doc_file, old_rootdir, new_rootdir):
		''' necessary when files are transferred from one file system to another, e.g., the root directory on Garibaldi is different from that on Guppy'''

		try:
			f = open(sel_doc_file, "r")
		except:
			apDisplay.printWarning("Unable to open file: %s" % sel_doc_file)
			return
		lines = f.readlines()
		newlines = []
		f.close()
			
		### replace old root directory with new root directory	
		count = 0
		for line in lines:
			if re.search(old_rootdir, line):
				newline = re.sub(old_rootdir, new_rootdir, line)
				count += 1
			else: 
				newline = line
			newlines.append(newline)
		if count > 0:
			apDisplay.printMsg("changing root directory from %s to %s in %s" % (old_rootdir, new_rootdir, sel_doc_file))
			f = open(sel_doc_file, "w")
			f.writelines(newlines)
			f.close()
		return
	def start(self):
						
		### determine which iterations to upload; last iter is defaulted to infinity
		uploadIterations = self.verifyUploadIterations()				
					
		### upload each iteration
		for iteration in uploadIterations:
			for j in range(self.runparams['NumberOfReferences']):
										
				### general error checking, these are the minimum files that are needed
				vol = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.mrc" % (self.params['timestamp'], iteration, j+1))
				particledatafile = os.path.join(self.resultspath, "particle_data_%s_it%.3d_vol%.3d.txt" % (self.params['timestamp'], iteration, j+1))
				if not os.path.isfile(vol):
					apDisplay.printError("you must have an mrc volume file in the 'external_package_results' directory")
				if not os.path.isfile(particledatafile):
					apDisplay.printError("you must have a particle data file in the 'external_package_results' directory")										
										
				### make chimera snapshot of volume
				try:
					self.createChimeraVolumeSnapshot(vol, iteration, j+1)
				except:
					apDisplay.printWarning("could not create Chimera volume snapshots")
				
				### instantiate database objects
				self.insertRefinementRunData(iteration, j+1)
				self.insertRefinementIterationData(iteration, j+1)
				
		### calculate Euler jumps
		self.calculateEulerJumpsAndGoodBadParticles(uploadIterations)			
Esempio n. 13
0
def getBoxSize(filename, msg=False):
	"""
	return boxsize of stack in pixels
	"""
	if not os.path.isfile(filename):
		if msg is True:
			apDisplay.printWarning("file does not exist")
		return (1,1,1)
	proc = subprocess.Popen("iminfo %s"%(filename), shell=True, stdout=subprocess.PIPE)
	proc.wait()
	lines = ""
	for line in proc.stdout:
		sline = line.strip()
		lines += line
		m = re.match("^Image\(s\) are ([0-9]+)x([0-9]+)x([0-9]+)", sline)	
		if m and m.groups() and len(m.groups()) > 1:
			xdim = int(m.groups()[0])
			ydim = int(m.groups()[1])
			zdim = int(m.groups()[2])
			return (xdim,ydim,zdim)
		m = re.match("^0\.\s+([0-9]+)x([0-9]+)\s+", sline)	
		if m and m.groups() and len(m.groups()) > 1:
			xdim = int(m.groups()[0])
			ydim = int(m.groups()[1])
			return (xdim,ydim,1)
		m = re.match("^0\.\s+([0-9]+)x([0-9]+)x([0-9]+)\s+", sline)	
		if m and m.groups() and len(m.groups()) > 1:
			xdim = int(m.groups()[0])
			ydim = int(m.groups()[1])
			zdim = int(m.groups()[2])
			return (xdim,ydim,zdim)
	if msg is True:
		apDisplay.printWarning("failed to get boxsize: "+lines)
	return (1,1,1)
	def readPartDocFile(self, reflist):
		partlist = []
		docfile = "part"+self.params['timestamp']+".doc"
		if not os.path.isfile(docfile) or apFile.fileSize(docfile) < 50:
			apDisplay.printWarning("Could not find doc file "+docfile+" to read particle angles")
			lastdocfile = "iter%03d/part%s_it%06d.doc"%(self.lastiter, self.params['timestamp'], self.lastiter)
			if not os.path.isfile(lastdocfile) or apFile.fileSize(lastdocfile) < 50:
				apDisplay.printError("Could not find backup doc file")
			apDisplay.printColor("Found a backup copy of docfile", "green")
			shutil.copy(lastdocfile, docfile)
		f = open(docfile, "r")
		mininplane = 360.0
		for line in f:
			if line[:2] == ' ;':
				continue
			spidict = operations.spiderInLine(line)
			origpartdict = self.spidict2partdict(spidict)
			partdict = self.adjustPartDict(origpartdict, reflist)
			if partdict['inplane'] < mininplane:
				mininplane = partdict['inplane']
			partlist.append(partdict)
		apDisplay.printMsg("minimum inplane: "+str(mininplane))
		for partdict in partlist:
			partdict['inplane'] = partdict['inplane']-mininplane
		apDisplay.printMsg("read rotation and shift parameters for "+str(len(partlist))+" particles")
		if len(partlist) < 1:
			apDisplay.printError("Did not find any particles in doc file: "+docfile)
		return partlist
	def importPicks(self, picks1, picks2, tight=False, msg=True):
		t0 = time.time()
		#print picks1
		#print self.currentpicks1
		curpicks1 = numpy.asarray(self.currentpicks1)
		curpicks2 = numpy.asarray(self.currentpicks2)
		#print curpicks1

		# get picks
		apTiltTransform.setPointsFromArrays(curpicks1, curpicks2, self.data)
		pixdiam = self.data['pixdiam']
		if tight is True:
			pixdiam /= 4.0
		#print self.data, pixdiam
		list1, list2 = apTiltTransform.alignPicks2(picks1, picks2, self.data, limit=pixdiam, msg=msg)
		if list1.shape[0] == 0 or list2.shape[0] == 0:
			apDisplay.printWarning("No new picks were found")

		# merge picks
		newpicks1, newpicks2 = apTiltTransform.betterMergePicks(curpicks1, list1, curpicks2, list2, msg=msg)
		newparts = newpicks1.shape[0] - curpicks1.shape[0]

		# copy over picks
		self.currentpicks1 = newpicks1
		self.currentpicks2 = newpicks2

		if msg is True:
			apDisplay.printMsg("Inserted "+str(newparts)+" new particles in "+apDisplay.timeString(time.time()-t0))

		return True
def readDictFromXml(filename):
	if not os.path.isfile(filename):
		apDisplay.printWarning("Failed to open file "+filename+" for reading")
		return None
	dom = xml.dom.minidom.parse(filename)
	xmldict = nodeToDict(dom.childNodes[0])
	return xmldict
 def _startLoop(self, info):
         """
         appionLoop OVERRIDE
         initilizes several parameters for a new image
         and checks if it is okay to start processing image
         """
         if info is None:
                 self.stats['lastimageskipped'] = True
                 self.stats['skipcount'] += 1
                 return False
         name = info['filename']
         # check to see if image of the same name is already in leginon
         imgq = leginon.leginondata.AcquisitionImageData(session=self.session, filename=name)
         results = imgq.query(readimages=False)
         if results:
                 apDisplay.printWarning("File %s.mrc exists at the destination" % name)
                 apDisplay.printWarning("Skip Uploading")
                 self.stats['lastimageskipped'] = True
                 self.stats['skipcount'] += 1
                 return False
         #calc images left
         self.stats['imagesleft'] = self.stats['imagecount'] - self.stats['count']
         #only if an image was processed last
         if(self.stats['lastcount'] != self.stats['count']):
                 apDisplay.printColor( "\nStarting image "+str(self.stats['count'])\
                         +" ( skip:"+str(self.stats['skipcount'])+", remain:"\
                         +str(self.stats['imagesleft'])+" ) file: "\
                         +apDisplay.short(name), "green")
                 self.stats['lastcount'] = self.stats['count']
                 self._checkMemLeak()
         # check to see if image has already been processed
         if self._alreadyProcessed(info):
                 return False
         self.stats['waittime'] = 0
         return True
Esempio n. 18
0
    def getResolutionData(self, avgpath, iternum):
        evenvol = os.path.join(avgpath, "avglist3_%dp1.mrc" % (self.params["rescut"]))
        oddvol = os.path.join(avgpath, "avglist3_%dp2.mrc" % (self.params["rescut"]))
        evenhed = os.path.join(avgpath, "avglist3_%dp1.hed" % (self.params["rescut"]))
        oddhed = os.path.join(avgpath, "avglist3_%dp2.hed" % (self.params["rescut"]))
        emancmd1 = "proc3d %s %s" % (evenvol, evenhed)
        apEMAN.executeEmanCmd(emancmd1, verbose=True)
        emancmd2 = "proc3d %s %s" % (oddvol, oddhed)
        apEMAN.executeEmanCmd(emancmd2, verbose=True)
        fscfile = "fsc.eotest.%d" % (iternum)
        fscpath = os.path.join(avgpath, fscfile)
        emancmd3 = "proc3d %s %s fsc=%s" % (evenhed, oddhed, fscpath)
        apEMAN.executeEmanCmd(emancmd3, verbose=True)

        if not os.path.isfile(fscpath):
            apDisplay.printWarning("Could not find FSC file: " + fscpath)
            return None
        f = open(fscpath, "r")
        xy = f.readlines()
        lines = len(xy)
        boxsize = lines * 2.0
        f.close()
        # calculate the resolution:
        halfres = apRecon.calcRes(fscpath, boxsize, self.params["step"])
        apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms" % (halfres), "cyan")

        # save to database
        resq = appiondata.ApResolutionData()
        resq["half"] = halfres
        resq["fscfile"] = fscpath

        return resq
	def processParticles(self, imgdata, partdatas, shiftdata):
		self.shortname = apDisplay.short(imgdata['filename'])

		### if only selected points along helix,
		### fill in points with helical step
		if self.params['helicalstep']:
			apix = apDatabase.getPixelSize(imgdata)
			partdatas = self.fillWithHelicalStep(partdatas, apix)

		### run batchboxer
		self.boxedpartdatas, self.imgstackfile, self.partmeantree = self.boxParticlesFromImage(imgdata, partdatas, shiftdata)
		if self.boxedpartdatas is None:
			self.stats['lastpeaks'] = 0
			apDisplay.printWarning("no particles were boxed from "+self.shortname+"\n")
			self.badprocess = True
			return None

		self.stats['lastpeaks'] = len(self.boxedpartdatas)

		apDisplay.printMsg("do not break function now otherwise it will corrupt stack")
		#time.sleep(1.0)

		### merge image particles into big stack
		totalpart = self.mergeImageStackIntoBigStack(self.imgstackfile, imgdata)

		### create a stack average every so often
		if self.stats['lastpeaks'] > 0:
			totalPartices = self.existingParticleNumber+self.stats['peaksum']+self.stats['lastpeaks']
			logpeaks = math.log(totalPartices)
			if logpeaks > self.logpeaks:
				self.logpeaks = math.ceil(logpeaks)
				numpeaks = math.ceil(math.exp(self.logpeaks))
				apDisplay.printMsg("writing averaging stack, next average at %d particles"%(numpeaks))
				mrc.write(self.summedParticles/float(totalPartices), "average.mrc")
		return totalpart
	def insertFSCData(self, iternum, refineIterData):
		fscfile = 'fsc.eotest.%d'%(iternum)
		fscpath = os.path.join(self.params['rundir'], "iter%03d"%(iternum), fscfile)

		if not os.path.isfile(fscpath):
			apDisplay.printWarning("Could not find FSC file: "+fscpath)
			return None

		f = open(fscpath, 'r')
		apDisplay.printMsg("inserting FSC Data into database")
		numinserts = 0
		for line in f:
			fscq = appiondata.ApFSCData()
			fscq['refineIter'] = refineIterData
			sline = line.strip()
			bits = sline.split('\t')
			fscq['pix'] = int(bits[0])
			fscq['value'] = float(bits[1])

			numinserts+=1
			if self.params['commit'] is True:
				fscq.insert()

		apDisplay.printMsg("inserted "+str(numinserts)+" rows of FSC data into database")
		f.close()
	def findLastCompletedIteration(self):
		#recondir = os.path.join(self.params['rundir'], "recon")
		recondir = self.params['rundir']
		iternum = 0
		stop = False
		while stop is False:
			## check if iteration is complete
			iternum += 1

			class1Volume = "recon_it%03d_half1_model.star"%(iternum)
			class2Volume = "recon_it%03d_half2_model.star"%(iternum)
			class1Volumepath = os.path.join(recondir,class1Volume)
			class2Volumepath = os.path.join(recondir,class2Volume)
			if not os.path.isfile(class1Volumepath) or not os.path.isfile(class2Volumepath):
				apDisplay.printWarning("Model.star file %s or %s is missing"%(class1Volumepath, class2Volumepath))
				stop = True
				break

		### set last working iteration
		numiter = iternum-1
		if numiter < 1:
			apDisplay.printError("No iterations were found")
		apDisplay.printColor("Found %d complete iterations"%(numiter), "green")

		return numiter	
 def readBatchUploadInfo(self):
         # in this example, the batch script file should be separated by tab
         # see example in function readUploadInfo for format
         batchfilename = self.params['batchscript']
         if not os.path.exists(batchfilename):
                 apDisplay.printError('Batch file %s not exist' % batchfilename)
                 return []
         batchfile = open(batchfilename,'r')
         lines = batchfile.readlines()
         batchfile.close()
         batchinfo = []
         count = 0
         for line in lines:
                 count += 1
                 #remove white space at ends
                 sline = line.strip()
                 if ' ' in sline:
                         apDisplay.printWarning("There is a space in the batch file on line %d"%(count))
                 #remove white space at ends
                 cols = sline.split('\t')
                 if len(cols) > 1:
                         batchinfo.append(cols)
                 else:
                         apDisplay.printWarning("Skipping line %d"%(count))
         return batchinfo
	def convertFSCFileForIteration(self, iteration):
		'''  RElion FSC info is in recon_it002_model.star and the final recon_model.star file, convert this to 3DEM format 
			 For the final iteration, we need to use recon_model.star rather than the iteration specific file to get the actual values.
		'''
	
		lastiter = self.findLastCompletedIteration() 
		if iteration == lastiter:
			fscfile = os.path.join(self.projmatchpath, "recon_model.star"%(iteration))
		else:
			fscfile = os.path.join(self.projmatchpath, "recon_it%.3d_half1_model.star"%(iteration))
			
		f = starFile.StarFile( fscfile )
		try: 
			f.read()
		except e:
			if iteration == lastiter:
				apDisplay.printWarning("%s file could not be opened, data will NOT be inserted into the database. Relion did NOT run to completion." % fscfile)
			else:
				apDisplay.printWarning("%s file could not be opened, data will NOT be inserted into the database" % fscfile)
			return False
		
		dataBlock = f.getDataBlock("data_model_class_1")
		loopDict = dataBlock.getLoopDict()

		newfscfile = open(os.path.join(self.resultspath, "recon_%s_it%.3d_vol001.fsc" % (self.params['timestamp'], iteration)), "w")
		newfscfile.write("### column (1) inverse Angstroms, column (2) Fourier Shell Correlation (FSC)")
		
		for valueSet in loopDict:
			resolution   = float(valueSet["_rlnResolution"]) # already in inverse angstrom
			fsc          = float(valueSet["_rlnGoldStandardFsc"]) # should go from a value of 1 to 0
			newfscfile.write("%.6f\t%.6f\n" % (resolution, fsc))
			
		newfscfile.close()
		
		return True
	def sortFolder(self):
		numsort = 0
		apDisplay.printMsg("Sorting files into clean folders")
		### move files for all particle iterations
		files = []
		for i in range(self.lastiter+1):
			iterdir = "iter%03d"%(i)
			apParam.createDirectory(iterdir, warning=False)
			wildcard = "part*_it*%03d_*.*"%(i)
			files.extend(glob.glob(wildcard))
			wildcard = "part*_it*%03d.*"%(i)
			files.extend(glob.glob(wildcard))
			for filename in files:
				if os.path.isfile(filename):
					numsort += 1
					shutil.move(filename,iterdir)
		if numsort < 3:
			apDisplay.printWarning("Problem in iteration file sorting, are they already sorted?")
		apDisplay.printMsg("Sorted "+str(numsort)+" iteration files")
		### move files for all reference iterations
		refsort = 0
		refdir = "refalign"
		apParam.createDirectory(refdir, warning=False)
		wildcard = "ref*_it*.*"
		files = glob.glob(wildcard)
		for filename in files:
			refsort += 1
			shutil.move(filename, refdir)
		#if refsort < 5:
		#	apDisplay.printError("Problem in reference file sorting")
		apDisplay.printMsg("Sorted "+str(refsort)+" reference files")
		return
def writeTiltFile(tiltfile, seriesname, imagedict, azimuth, refimg ):
        '''
        Create the geometry object (tilt angles, dir of tilt axis, origins of ROI) ---
        1. read the params from the database 
        2. write params to text file
        '''


        try:
                f = open(tiltfile,'w')
        except:
                apDisplay.printWarning("Failed to create %s for writing the protomo2 tilt geometry parameter file" % (tiltfile, ))
                raise

        f.write('\n')


        f.write ("TILT SERIES %s\n" % seriesname)
        f.write ("\n") 
        f.write ("   AXIS\n")
        f.write ("\n")
        f.write ("     TILT AZIMUTH    %g\n" % azimuth)
        f.write ("\n")

        keys = imagedict.keys()
        keys.sort()
        for n in keys:
                f.write ("   IMAGE %-5d     FILE %s       ORIGIN [ %8.3f %8.3f ]    TILT ANGLE    %8.3f    ROTATION     %8.3f\n" % (n, imagedict[n]['filename'], imagedict[n]['x'], imagedict[n]['y'], imagedict[n]['tilt'], imagedict[n]['rotation']))

        f.write ("\n")
        f.write ("   REFERENCE IMAGE %d\n" % refimg)
        f.write ("\n")
        f.write ("\n")
        f.write (" END\n")
        f.close()
	def start(self):
		print self.params
		if self.params['clusterId'] is not None:
			self.clusterdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterId'])
			self.useClusterForTemplateStack()
		elif self.params['alignId'] is not None:
			self.aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignId'])
			self.useClusterForTemplateStack()
		else:
			apDisplay.printMsg("Using local file: '"+str(self.params['templatestack'])+"' to upload template")

			# copy templates to final location
			if str(self.params['templatestack'])[-4:] == ".img" or str(self.params['templatestack'])[-4:] == ".hed":
				self.params['templatestack'] = self.params['templatestack'][:-4]
			if str(self.params['runname'])[-4:] == ".img" or str(self.params['runname'])[-4:0] == ".hed":
				self.params['runname'] = self.params['runname'][:-4]
			shutil.copyfile(str(self.params['templatestack'])+".img", os.path.join(self.params['rundir'], str(self.params['runname'])+".img"))
			shutil.copyfile(str(self.params['templatestack'])+".hed", os.path.join(self.params['rundir'], str(self.params['runname'])+".hed"))
			self.numimages = apFile.numImagesInStack(os.path.join(self.params['rundir'], str(self.params['runname'])+".hed"))

		# insert templates to database
		if self.params['commit'] is True:
			self.uploadTemplateStack(insert=True)
		else:
			apDisplay.printWarning("not committing results to DB")
def writeTileFile2(tiltfile, seriesname, imagelist, origins, tilts, azimuth, refimg):
        
        try:
                f = open(tiltfile,'w')
        except:
                apDisplay.printWarning("Failed to create %s for writing the protomo2 tilt geometry parameter file" % (tiltfile, ))
                raise
        
        f.write('\n')


        f.write ("TILT SERIES %s\n" % seriesname)
        f.write ("\n") 
        f.write ("   AXIS\n")
        f.write ("\n")
        f.write ("     TILT AZIMUTH    %g\n" % azimuth)
        f.write ("\n")
        for n in range(len(imagelist)):
                imagename=os.path.splitext(os.path.basename(imagelist[n]))[0] #strip off path and extension from mrc file
                f.write ("   IMAGE %-5d     FILE %s       ORIGIN [ %8.3f %8.3f ]    TILT ANGLE    %8.3f    ROTATION     %8.3f\n" % (n+1, imagename, origins[n]['x'], origins[n]['y'], tilts[n], 0))

        f.write ("\n")
        f.write ("   REFERENCE IMAGE %d\n" % refimg)
        f.write ("\n")
        f.write ("\n")
        f.write (" END\n")
        f.close()
	def loopCommitToDatabase(self, imgdata):
		"""
		Over-writes the particleLoop commit and uses the appionLoop commit
		"""
		tiltdata = apTiltPair.getTiltPair(imgdata)
		if tiltdata is None:
			apDisplay.printWarning("Tilt data not found; not commiting data")
			return False

		### insert the runid
		self.commitRunToDatabase(imgdata['session'], True)

		if self.assess is not None:
			#note runid is overrided to be 'run1'
			apDatabase.insertImgAssessmentStatus(imgdata, self.params['runname'], self.assess)
			apDatabase.insertImgAssessmentStatus(tiltdata, self.params['runname'], self.assess)

		if len(self.peaktree1) < 3 or len(self.peaktree1) < 3:
			apDisplay.printWarning("Not enough particle picks; not commiting transform or particle data")
			return False

		### insert the transform
		transdata = apTiltPair.insertTiltTransform(imgdata, tiltdata, self.tiltparams, self.params)
		### insert the particles
		self.insertParticlePeakPairs(imgdata, tiltdata, transdata)
	def start(self):
		sessionname = self.params['sessionname']
		runname = self.params['runname']
		preset = self.params['preset']

		sessionq = leginondata.SessionData(name=sessionname)
		presetq=leginondata.PresetData(name=preset)
		imgquery = leginondata.AcquisitionImageData()
		imgquery['preset']  = presetq
		imgquery['session'] = sessionq
		imgtree = imgquery.query(readimages=False)
		partq = appiondata.ApContourData()
		sessiond = sessionq.query()
		selectionid = apParticle.getSelectionIdFromName(runname, sessionname)
		if not selectionid:
			apDisplay.printWarning('No Object Tracing Run found in database, Skipping.......')
			return
		selectionrundata = apParticle.getSelectionRunDataFromID(selectionid)


		file = open('contourpickerData-' + sessionname + '.txt','w')
		file.write('session_id ' + runname + '\n')
		file.write('usr_id ' + os.getlogin() + '\n')
		file.write('experiment_name ' + sessionname + '\n')
		file.write('experiment_description ' + sessiond[0]['comment'].strip() + '\n')
		file.write('nimages ' + str(len(imgtree)) + '\n')

		for imgdata in imgtree:
			file.write('START_IMAGE' + '\n')
			partq['image'] = imgdata
			partq['selectionrun'] = selectionrundata
			partd = partq.query()
			if len(partd)>0:
				file.write('image_refID ' + str(partd[0]['image'].dbid) + '\n') 
			file.write('image_name ' + imgdata['filename'] + '\n') 
			if len(partd)>0:
				file.write('time_roi ' + str(partd[0].timestamp) + '\n') 
			#file.write('time_roi ' + partd[0]['DEF_timestamp'] + '\n') 
			file.write('dfac = 1\n')
			maxversion = 0
			numparticles = 0
			for part in partd:
				if int(part['version'])>maxversion:
					maxversion = int(part['version'])
			for part in partd:
				if int(part['version'])==maxversion:
					numparticles+=1
			file.write('version_id ' + str(maxversion) + '\n')
			file.write('ncontours ' + str(numparticles) + '\n')
			pointq = appiondata.ApContourPointData()
			for part in partd:
				if int(part['version'])==maxversion:
			#		file.write('contour_number ' + )
					file.write('method_used ' + part['method'] + ' ')
					pointq['contour'] = part
					pointd = pointq.query()
					for point in pointd:
						file.write(str(point['x']) + ',' + str(point['y']) + ';')
					file.write('\n')
			file.write('END_IMAGE' + '\n')
	def start(self):
		### universal particle counter
		self.partnum = 1

		### final stack file
		self.combinefile = os.path.join( self.params['rundir'], self.params['stackfilename'] )
		if os.path.isfile(self.combinefile):
			apDisplay.printError("A stack with name "+self.params['stackfilename']+" and path "
				+self.params['rundir']+" already exists.")

		### loop through stacks
		for stackstr in self.params['stackids']:
			stackid = int(stackstr)

			### get stack data
			stackdata = apStack.getOnlyStackData(stackid)

			### append particle to stack file
			self.appendToStack(stackdata)

			if self.params['commit'] is True:
				### insert stack data
				apDisplay.printColor("inserting new stack particles from stackid="+str(stackid), "cyan")
				self.commitStack(stackid)
			else:
				apDisplay.printWarning("not committing data to database")

		apStack.averageStack(stack=self.combinefile)
Esempio n. 31
0
def averageSubStack(partlist, stackfile, bin=1):
        if len(partlist) > 300:
                partlist = partlist[:300]
        boxsize = apImagicFile.getBoxsize(stackfile)
        if len(partlist) == 0:
                binboxsize = boxsize/bin
                blank = numpy.ones((binboxsize, binboxsize), dtype=numpy.float32)
                return blank
        if not os.path.isfile(stackfile):
                apDisplay.printWarning("could not find stack, "+stackfile)
                return False
        partdatalist = apImagicFile.readParticleListFromStack(stackfile, partlist, boxsize, msg=False)
        partdataarray = numpy.asarray(partdatalist)
        finaldata = partdataarray.mean(0)
        if bin > 1:
                finaldata = apImage.binImg(finaldata, bin)
        return finaldata
    def _initializeDoneDict(self):
        """
		reads or creates a done dictionary
		"""
        self.donedictfile = os.path.join(self.params['rundir'],
                                         self.functionname + ".donedict")
        if os.path.isfile(
                self.donedictfile) and self.params['continue'] == True:
            ### unpickle previously done dictionary
            apDisplay.printMsg("Reading old done dictionary: " +
                               os.path.basename(self.donedictfile))
            f = open(self.donedictfile, 'r')
            self.donedict = cPickle.load(f)
            f.close()
            try:
                if self.donedict['commit'] == self.params['commit']:
                    ### all is well
                    apDisplay.printMsg("Found " + str(len(self.donedict)) +
                                       " done dictionary entries")
                    return
                elif self.donedict['commit'] is True and self.params[
                        'commit'] is not True:
                    ### die
                    apDisplay.printError(
                        "Commit flag was enabled and is now disabled, create a new runname"
                    )
                else:
                    ### set up fresh dictionary
                    apDisplay.printWarning(
                        "'--commit' flag was changed, creating new done dictionary"
                    )
            except KeyError:
                apDisplay.printMsg("Found " + str(len(self.donedict)) +
                                   " done dictionary entries")

        ### set up fresh dictionary
        self.donedict = {}
        self.donedict['commit'] = self.params['commit']
        apDisplay.printMsg("Creating new done dictionary: " +
                           os.path.basename(self.donedictfile))

        ### write donedict to file
        f = open(self.donedictfile, 'w', 0666)
        cPickle.dump(self.donedict, f)
        f.close()
        return
Esempio n. 33
0
    def insertCoranRun(self, insert=False):
        # create a AlignAnalysisRun object
        analysisq = appiondata.ApAlignAnalysisRunData()
        analysisq['runname'] = self.params['runname']
        analysisq['path'] = appiondata.ApPathData(
            path=os.path.abspath(self.params['rundir']))
        # ... path makes the run unique:
        uniquerun = analysisq.query(results=1)
        if uniquerun and insert is True:
            apDisplay.printError("Run name '"+self.params['runname']+"' for align stack id="+\
             str(self.params['alignstackid'])+"\nis already in the database")

        coranq = appiondata.ApCoranRunData()
        coranq['num_factors'] = self.params['numfactors']
        coranq['mask_diam'] = 2.0 * self.params['maskrad']
        coranq['run_seconds'] = self.runtime

        # finish AlignAnalysisRun object
        analysisq['description'] = self.params['description']
        analysisq['alignstack'] = self.alignstackdata
        analysisq['hidden'] = False
        analysisq['coranrun'] = coranq

        apDisplay.printMsg(
            "inserting Align Analysis Run parameters into database")
        if insert is True:
            analysisq.insert()

        ### eigen data
        for i in range(self.params['numfactors']):
            factnum = i + 1
            eigenq = appiondata.ApCoranEigenImageData()
            eigenq['coranRun'] = coranq
            eigenq['factor_num'] = factnum
            path = os.path.join(self.params['rundir'], "coran")
            eigenq['path'] = appiondata.ApPathData(path=os.path.abspath(path))
            imgname = ("eigenimg%02d.png" % (factnum))
            eigenq['image_name'] = imgname
            if not os.path.isfile(os.path.join(path, imgname)):
                apDisplay.printWarning(imgname + " does not exist")
                continue
            if insert is True:
                eigenq['percent_contrib'] = self.contriblist[i]
                eigenq.insert()

        return
Esempio n. 34
0
 def checkConflicts(self):
     super(MakeDDParticleMovieLoop, self).checkConflicts()
     if self.params['bin'] != 1:
         apDisplay.printError(
             'binning is not yet implemented, please use bin=1')
     #if self.params['denoise'] and not self.params['ddstack']:
     #       apDisplay.printError('denoise only works with ddstack')
     if self.params['denoise'] and self.params['framestep'] > 1:
         apDisplay.printWarning(
             'Interval of frames must be one when denoising')
         apDisplay.printWarning('Forcing it to 1....')
         self.params['framestep'] = 1
     if not self.params['nframe']:
         # force nframe to a large member so that checkIsDD will consider it as dd data
         # This number will limit the total processing frames if smaller than
         # the actual number of frames in the images
         self.params['nframe'] = 1000
Esempio n. 35
0
def moveStack(filename1, filename2, warn=True):
    ### replace one imagic stack with another
    rootname1 = os.path.splitext(filename1)[0]
    rootname2 = os.path.splitext(filename2)[0]
    for ext in (".hed", ".img"):
        inf = rootname1 + ext
        outf = rootname2 + ext
        if os.path.isfile(inf):
            if warn is True:
                apDisplay.printWarning("replacing stack file '%s' with '%s' " %
                                       (outf, inf))
                time.sleep(1)
            try:
                shutil.move(inf, outf)
            except:
                apDisplay.printWarning('%s could not be replaced with %s' %
                                       (outf, inf))
def totalLeastSquares(X, Y, W=None, epsilon=1e-5, maxiter=500):
    """
        iterative refine weights of observations, de-emphasizing bad fitting points
        """
    t0 = time.time()
    ### check the input
    if checkMatrixSizes(X, Y) is False:
        return None

    ### setup weights if necessary
    if W is None:
        #W = numpy.random.random((m)) # weights for the observations
        #W = W*float(M)/W.sum()
        W = numpy.ones(Y.shape)  # even weights for the observations

    ### solve it
    err0 = None
    sys.stderr.write("running total least squares")
    for i in range(maxiter):
        sys.stderr.write(".")
        beta = weightedLeastSquares(X, Y, W)
        if beta is None:
            if i < 2:
                return None
            beta = weightedLeastSquaresSVD(X, Y, W)

        ## calculate the absolute mean error
        err = numpy.absolute(numpy.dot(X, beta) - Y).ravel()
        #print "totalLeastSquares iter %d error: %.4f"%(i, err.mean())
        ## fit to a normal distribution
        normerr = (err - err.min()) / err.std()
        ## calculate new weights based on
        W = numpy.exp(-1 * normerr**2)
        if W.sum() == 0:
            apDisplay.printWarning("Failed to set weights")
            return beta
        ## see if we can stop
        if err0 is not None:
            change = numpy.absolute(err - err0).mean()
            if change < epsilon:
                break
        err0 = err
    apDisplay.printMsg("totalLeastSquares completed in %s in %d iterations" %
                       (apDisplay.timeString(time.time() - t0), i))
    return beta
Esempio n. 37
0
	def getTargets(self, imgdata, scratchdir='', handlefiles='direct'):
		targetdict={}
		#copy flatfields
		
		brightrefpath=imgdata['bright']['session']['image path']
		brightrefname=imgdata['bright']['filename']+'.mrc'
		brightref=os.path.join(brightrefpath,brightrefname)
		
		darkrefpath=imgdata['dark']['session']['image path']
		darkrefname=imgdata['dark']['filename']+'.mrc'
		darkref=os.path.join(darkrefpath,darkrefname)
		
		#################################### do away with override flatfield option
		framesdirname=imgdata['filename']+'.frames'
		apDisplay.printMsg('Copying frames %s' % (framesdirname))
		framespath=imgdata['session']['frame path']
		framespathname=os.path.join(framespath,framesdirname)
	
		if handlefiles == 'direct':
			targetdict['brightref']=brightref
			targetdict['darkref']=darkref
			targetdict['framespathname']=framespathname
			targetdict['outpath']=self.params['rundir']
		elif handlefiles == 'copy':
			shutil.copy(brightref,scratchdir)
			shutil.copy(darkref,scratchdir)
			targetdict['brightref']=os.path.join(scratchdir,brightrefname)
			targetdict['darkref']=os.path.join(scratchdir,darkrefname)
			try:
				shutil.copytree(framespathname,os.path.join(scratchdir, framesdirname))
			except:
				apDisplay.printWarning('there was a problem copying the frames for %s' % (imgdata['filename']))
			targetdict['framespathname']=os.path.join(scratchdir,framesdirname)
			targetdict['outpath']=os.path.join(scratchdir,imgdata['filename'])
			
		elif handlefiles == 'link':
			os.symlink(brightref,os.path.join(scratchdir,brightrefname))
			os.symlink(darkref,os.path.join(scratchdir,darkrefname))
			os.symlink(framespathname,os.path.join(scratchdir, framesdirname))
			
			targetdict['brightref']=os.path.join(scratchdir,brightrefname)
			targetdict['darkref']=os.path.join(scratchdir,darkrefname)
			targetdict['framespathname']=os.path.join(scratchdir,framesdirname)
			targetdict['outpath']=os.path.join(scratchdir,imgdata['filename'])
		return targetdict
    def start(self):
        ### get volume files
        volumefiles = glob.glob(self.params['volumes'])
        if not volumefiles:
            apDisplay.printError("Could not find volumes, %s" %
                                 (self.params['volumes']))

        ### make list of all alignments to run
        cmdlist = []
        alignfiles = []
        for volfile in volumefiles:
            alignfile = "align-" + os.path.basename(volfile)
            alignfiles.append(alignfile)
            emancmd = "align3d %s %s %s " % (self.params['reference'], volfile,
                                             alignfile)
            if self.params['slow'] is True:
                emancmd += "slow "
            if self.params['noshrink'] is True:
                emancmd += "noshrink "
            #print emancmd
            cmdlist.append(emancmd)

        ### run several alignment commands in parallel
        apThread.threadCommands(cmdlist)

        ### average volumes together
        ref = mrc.read(self.params['reference'])
        average = numpy.zeros(ref.shape, dtype=numpy.float32)
        del ref
        count = 0
        for alignfile in alignfiles:
            if not os.path.isfile(alignfile):
                apDisplay.printWarning("aligned volume not found: %s" %
                                       (alignfile))
            aligned = mrc.read(alignfile)
            count += 1
            ### this assume all aligned volume have same box size
            average += aligned
            del aligned

        ### save average
        average /= count
        avgfile = os.path.abspath(self.params['average'])
        mrc.write(average, avgfile)
        apDisplay.printMsg("Wrote average file: " + avgfile)
Esempio n. 39
0
def runCTFdisplayTools(imgdata,
                       ctfvalues,
                       opimagedir,
                       fftpath=None,
                       fftfreq=None):
    ### RUN CTF DISPLAY TOOLS
    ctfdisplaydict = ctfdisplay.makeCtfImages(imgdata, ctfvalues, fftpath,
                                              fftfreq)
    if ctfdisplaydict is None:
        return ctfvalues
    ### save the classic images as well
    if 'graph1' in ctfvalues:
        ctfvalues['graph3'] = os.path.basename(ctfvalues['graph1'])
    if 'graph2' in ctfvalues:
        ctfvalues['graph4'] = os.path.basename(ctfvalues['graph2'])
    ### new powerspec file
    psfile = os.path.join(opimagedir, ctfdisplaydict['powerspecfile'])
    if not os.path.isfile(ctfdisplaydict['powerspecfile']):
        apDisplay.printWarning("Powerspec file not created")
    else:
        print ctfdisplaydict['powerspecfile']
        shutil.move(ctfdisplaydict['powerspecfile'], psfile)
        ctfvalues['graph1'] = os.path.basename(psfile)
    ### new 1d plot file
    plotfile = os.path.join(opimagedir, ctfdisplaydict['plotsfile'])
    shutil.move(ctfdisplaydict['plotsfile'], plotfile)
    ctfvalues['graph2'] = os.path.basename(plotfile)
    ctfvalues['confidence_30_10'] = ctfdisplaydict['conf3010']
    ctfvalues['confidence_5_peak'] = ctfdisplaydict['conf5peak']
    ctfvalues['overfocus_conf_30_10'] = ctfdisplaydict['overconf3010']
    ctfvalues['overfocus_conf_5_peak'] = ctfdisplaydict['overconf5peak']
    ctfvalues['resolution_80_percent'] = ctfdisplaydict['res80']
    ctfvalues['resolution_50_percent'] = ctfdisplaydict['res50']
    if not 'confidence_d' in ctfvalues or ctfvalues['confidence_d'] is None:
        ctfvalues['confidence_d'] = ctfdisplaydict['conf5peak']
    if not 'confidence' in ctfvalues or ctfvalues['confidence'] is None:
        ctfvalues['confidence'] = ctfdisplaydict['conf3010']

    ### override the confidence
    ctfvalues['confidence'] = max(ctfvalues['confidence'],
                                  ctfvalues['confidence_d'],
                                  ctfdisplaydict['conf5peak'],
                                  ctfdisplaydict['conf3010'])

    return ctfvalues
    def _removeProcessedImages(self):
        startlen = len(self.imgtree)
        donecount = 0
        reproccount = 0
        rejectcount = 0
        tiltcount = 0
        self.stats['skipcount'] = 0
        newimgtree = []
        count = 0
        t0 = time.time()
        for imgdata in self.imgtree:
            count += 1
            if count % 10 == 0:
                sys.stderr.write(".")
            skip, reason = self.skipTestOnImage(imgdata)
            imgname = imgdata['filename']
            if skip is True:
                if reason == 'reproc':
                    reproccount += 1
                elif reason == 'reject' or reason == 'tilt':
                    self._writeDoneDict(imgname)
                    rejectcount += 1

            if skip is True:
                if self.stats['skipcount'] == 0:
                    sys.stderr.write("skipping processed images\n")
                elif self.stats['skipcount'] % 80 == 0:
                    sys.stderr.write(".\n")
                else:
                    sys.stderr.write(".")
                self.stats['skipcount'] += 1
            else:
                newimgtree.append(imgdata)
        sys.stderr.write("\n")
        apDisplay.printMsg("finished skipping in %s" %
                           (apDisplay.timeString(time.time() - t0)))
        if self.stats['skipcount'] > 0:
            self.imgtree = newimgtree
            sys.stderr.write("\n")
            apDisplay.printWarning("skipped " + str(self.stats['skipcount']) +
                                   " of " + str(startlen) + " images")
            apDisplay.printMsg("[[ " + str(reproccount) + " no reprocess " +
                               " | " + str(rejectcount) + " rejected " +
                               " | " + str(tiltcount) + " wrong tilt " +
                               " | " + str(donecount) + " in donedict ]]")
Esempio n. 41
0
 def checkMPI(self):
         mpiexe = apParam.getExecPath("mpirun")
         if mpiexe is None:
                 return None
         xmippexe = apParam.getExecPath("xmipp_mpi_ml_refine3d")
         if xmippexe is None:
                 return None
         lddcmd = "ldd "+xmippexe+" | grep mpi"
         proc = subprocess.Popen(lddcmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         proc.wait()
         lines = proc.stdout.readlines()
         #print "lines=", lines
         if lines and len(lines) > 0:
                 return mpiexe
         else:
                 apDisplay.printWarning("Failed to find mpirun")
                 print "lines=", lines
                 return None
Esempio n. 42
0
	def getResolutionData(self, iternum):
		fscfile = 'fsc.eotest.%d'%(iternum)
		fscpath = os.path.join(self.params['rundir'], "iter%03d"%(iternum), fscfile)

		if not os.path.isfile(fscpath):
			apDisplay.printWarning("Could not find FSC file: "+fscpath)
			return None

		# calculate the resolution:
		halfres = apRecon.calcRes(fscpath, self.boxsize, self.apix)
		apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms"%(halfres), "cyan")

		# save to database
		resq=appiondata.ApResolutionData()
		resq['half'] = halfres
		resq['fscfile'] = fscfile

		return resq
	def getRMeasureData(self, iteration):
		volumeDensity='threed.'+iteration['num']+'a.mrc'

		volPath = os.path.join(self.params['rundir'], volumeDensity)
		if not os.path.exists(volPath):
			apDisplay.printWarning("R Measure failed, volume density not found: "+volPath)
			return None

		resolution = apRecon.runRMeasure(self.params['apix'], volPath)

		if resolution is None:
			return None

		rmesq=appiondata.ApRMeasureData()
		rmesq['volume']=volumeDensity
		rmesq['rMeasure']=resolution

		return rmesq
Esempio n. 44
0
 def processParticle(self, partarray):
     if self.filetype == "mrc":
         partfile = os.path.join(self.partdir,
                                 "part%06d.mrc" % (self.index))
         mrc.write(partarray, partfile)
     else:
         partfile = os.path.join(self.partdir,
                                 "part%06d.spi" % (self.index))
         try:
             spider.write(partarray, partfile)
         except:
             print partarray
             apDisplay.printWarning(
                 "failed to write spider file for part index %d" %
                 (self.index))
             print partarray.shape
             spider.write(partarray, partfile)
     self.partdocf.write(os.path.abspath(partfile) + " 1\n")
	def setupRunDirectory(self):
		"""
		Set the run directory
		"""
		if self.params['rundir'] is None:
			apDisplay.printWarning("run directory not defined, automatically setting it")
			self.setProcessingDirName()
			self.setRunDir()
			if self.params['rundir'] is None:
				apDisplay.printError("No run directory was set")

		if self.quiet is False:
			apDisplay.printMsg("Run directory: "+self.params['rundir'])

		if not os.path.isdir(self.params['rundir']):
				apDisplay.printError("run directory must exist for FileScript run")

		os.chdir(self.params['rundir'])
 def checkRefineLogError(self):
     f = open(self.tasklogfile)
     text = f.read()
     f.close()
     lines = text.split('\n')
     f.close()
     tasklog_basename = os.path.basename(self.tasklogfile)
     # check alarm and error
     if 'ERROR' in text:
         self.failed = True
         apDisplay.printError('There are Errors in %s' % (tasklog_basename),
                              False)
     if 'WARNING' in text:
         apDisplay.printWarning('There are Warning in %s during %s' %
                                (tasklog_basename, self.task_noun))
         for line in lines:
             if 'WARNING' in line:
                 apDisplay.printWarning(line)
def getRgbFile(msg=True):
	"""
	This file comes with xorg-x11-server-Xorg in Fedora 7,8
	missing in Fedora 9
	"""
	filelist = [
		"/usr/share/X11/rgb",
		"/usr/X11R6/lib64/X11/rgb",
		"/usr/X11R6/lib/X11/rgb",
	]
	xversion = getXversion()
	if xversion is None or xversion > 1.0109:
		return " "
	for rgbfile in filelist:
		if os.path.isfile(rgbfile+".txt"):
			return " -co "+rgbfile
	apDisplay.printWarning("Xvfb: could not find RGB File")
	return " "
Esempio n. 48
0
 def _getRefImageData(self, reftype):
     imagedata = super(GatanK2Processing, self).getCorrectedImageData()
     if not self.use_full_raw_area:
         refdata = imagedata[reftype]
         if refdata is None:
             # Use chosen default image
             apDisplay.printWarning(
                 'No %s reference for the image, use default' % reftype)
             default_image = self.getCorrectedImageData()
             refdata = default_image[reftype]
     else:
         # use most recent CorrectorImageData
         # TO DO: this should research only ones before the image is taken.
         scopedata = self.image['scope']
         channel = self.image['channel']
         refdata = self.c_client.researchCorrectorImageData(
             reftype, scopedata, self.camerainfo, channel)
     return refdata
Esempio n. 49
0
 def checkExistingFile(self):
     savedtomopath = os.path.join(self.params['rundir'],
                                  self.params['name'] + ".rec")
     origtomopath = self.params['file']
     apDisplay.printWarning(
         "A Tomogram by the same filename already exists: '" +
         savedtomopath + "'")
     ### a tomogram by the same name already exists
     savedheader = mrc.readHeaderFromFile(savedtomopath)
     savedshape = savedheader['shape']
     origheader = self.origheader
     origshape = self.origshape
     order = self.params['order']
     if self.params['full']:
         newshape = (self.imageshape[0], origshape[order.find('Z')],
                     self.imageshape[1])
     else:
         newshape = (origshape[order.find('X')], origshape[order.find('Y')],
                     origshape[order.find('Z')])
     # not using md5 for file comparison because it takes too long.  With padding, only min and max are stable
     if newshape != savedshape or savedheader['amax'] != origheader[
             'amax'] or savedheader['amin'] != origheader['amin']:
         different = True
     else:
         different = False
         mdnew = None
     if different:
         apDisplay.printWarning(
             "The tomograms are different, cannot overwrite")
         return True
     elif apDatabase.isTomoInDB(mdnew, self.params['full'], savedtomopath):
         ### they are the same and its in the database already
         apDisplay.printWarning(
             "Identical Tomogram already exists in the database!")
         apDisplay.printWarning("Upload Not Allowed")
         self.params['commit'] = False
         return True
     else:
         ### they are the same, but it is not in the database
         apDisplay.printWarning("The same tomogram with name '" +
                                savedtomopath +
                                "' already exists, but is not uploaded!")
         if self.params['commit'] is True:
             apDisplay.printMsg("Inserting tomogram into database...")
Esempio n. 50
0
def readData(filename, filetype=None):
    """
	savedata = {}
	savedata['theta'] = self.data['theta']
	savedata['gamma'] = self.data['gamma']
	savedata['phi'] = self.data['phi']
	savedata['shiftx'] = self.data['shiftx']
	savedata['shifty'] = self.data['shifty']
	savedata['savetime'] = time.asctime()+" "+time.tzname[time.daylight]
	savedata['filetype'] = tiltfile.filetypes[self.data['filetypeindex']]
	savedata['picks1'] = self.getArray1()
	savedata['picks2'] = self.getArray2()
	savedata['align1'] = self.getAlignedArray1()
	savedata['align2'] = self.getAlignedArray2()
	savedata['rmsd'] = self.getRmsdArray()
	savedata['filepath'] = os.path.join(self.data['dirname'], self.data['outfile'])
	savedata['image1name'] = self.panel1.filename
	savedata['image2name'] = self.panel2.filename
	"""
    if not os.path.isfile(filename):
        apDisplay.printWarning("file " + filename + " does not exist")
        return None

    if filetype is None:
        filetype = guessFileType(filename)

    if filetype == 'text':
        savedata = readFromTextFile(filename)
    elif filetype == 'xml':
        savedata = readFromXMLFile(filename)
    elif filetype == 'spider':
        savedata = readFromSpiderFile(filename)
    elif filetype == 'pickle':
        savedata = readFromPickleFile(filename)
    else:
        apDisplay.printWarning("unknown file type")
        return None

    savedata['filename'] = os.path.basename(filename)
    savedata['loadtime'] = time.asctime() + " " + time.tzname[time.daylight]
    savedata['shiftx'] = 0.0
    savedata['shifty'] = 0.0
    savedata['scale'] = 1.0
    return savedata
Esempio n. 51
0
def getBestCtfValue(imgdata, sortType='res80', method=None, msg=True):
	"""
	takes an image and get the best ctfvalues for that image
	"""
	### get all ctf values
	ctfq = appiondata.ApCtfData()
	ctfq['image'] = imgdata
	ctfvalues = ctfq.query()
	imgname = apDisplay.short(imgdata['filename'])

	if msg is True:
		print "Found %d ctf values"%(len(ctfvalues))

	### check if it has values
	if ctfvalues is None:
		apDisplay.printWarning("no CTF values found in database for img %s"%(imgname))
		return None

	### find the best values
	bestsortvalue = -1
	bestctfvalue = None
	for ctfvalue in ctfvalues:
		if method is not None:
			imgmethod = getCtfMethod(ctfvalue)
			if method != imgmethod:
				continue
		sortvalue = getSortValueFromCtfQuery(ctfvalue, sortType)
		if sortvalue is None:
			continue
		if msg is True:
			print "%.3f -- %s"%(sortvalue, ctfvalue['acerun']['name'])
		if sortvalue > bestsortvalue:
			bestsortvalue = sortvalue
			bestctfvalue = ctfvalue

	if bestctfvalue is None:
		apDisplay.printWarning("no best CTF value for image %s"%(imgname))
		return None

	if msg is True:
		print "*** %.3f"%(bestsortvalue)
		printCtfData(bestctfvalue)

	return bestctfvalue
 def checkIsDD(self):
     apDisplay.printWarning('Checking for dd')
     if self.params['ddstack'] > 0:
         self.is_dd_stack = True
         self.is_dd = True
     else:
         if self.params['preset'] and '-a' in self.params['preset'] and (
                 self.params['nframe'] or self.params['driftlimit'] > 0):
             self.is_dd = True
             self.is_dd_stack = True
         elif self.params['mrcnames'] and self.params['mrcnames'].split(
                 ',')[0] and '-a' in self.params['mrcnames'].split(
                     ',')[0] and (self.params['nframe']
                                  or self.params['driftlimit'] > 0):
             self.is_dd = True
             self.is_dd_stack = True
         elif self.params['nframe']:
             self.is_dd = True
             self.is_dd_frame = True
Esempio n. 53
0
def getRMeasurePath():
	unames = os.uname()
	if unames[-1].find('64') >= 0:
		exename = 'rmeasure64.exe'
	else:
		exename = 'rmeasure32.exe'
	rmeasexe = subprocess.Popen("which "+exename, shell=True, stdout=subprocess.PIPE).stdout.read().strip()
	if not os.path.isfile(rmeasexe):
		rmeasexe = os.path.join(apParam.getAppionDirectory(), 'bin', exename)
	if not os.path.isfile(rmeasexe):
		exename = "rmeasure.exe"
		rmeasexe = subprocess.Popen("which "+exename, shell=True, stdout=subprocess.PIPE).stdout.read().strip()
	if not os.path.isfile(rmeasexe):
		exename = "rmeasure"
		rmeasexe = subprocess.Popen("which "+exename, shell=True, stdout=subprocess.PIPE).stdout.read().strip()
	if not os.path.isfile(rmeasexe):
		apDisplay.printWarning(exename+" was not found at: "+apParam.getAppionDirectory())
		return None
	return rmeasexe
Esempio n. 54
0
 def checkConflicts(self):
     # Check that the program needed exists
     gpuexelist = ['dosefgpu_driftcorr']
     exename = 'dosefgpu_driftcorr'
     gpuexe = subprocess.Popen(
         "which " + exename, shell=True,
         stdout=subprocess.PIPE).stdout.read().strip()
     if not os.path.isfile(gpuexe):
         apDisplay.printError('Correction program "%s" not available' %
                              exename)
     # We don't have gpu locking
     if self.params['parallel']:
         apDisplay.printWarning(
             'Make sure that you use different gpuid for each parallel process'
         )
     if self.params['stackid'] and not self.params['ddstack']:
         apDisplay.printError(
             'Specify stack alone does not work.  Use makeDDRawFrameStack.py instead'
         )
def highPassFilter(imgarray, apix=1.0, bin=1, radius=0.0, localbin=8, msg=True):
        """
        high pass filter image to radius resolution
        """
        if radius is None or radius < 1 or imgarray.shape[0] < 256:
                if msg is True:
                        apDisplay.printMsg("skipping high pass filter")
                return(imgarray)
        try:
                bimgarray = binImg(imgarray, localbin)
                sigma=float(radius/apix/float(bin*localbin))
                filtimg = ndimage.gaussian_filter(bimgarray, sigma=sigma)
                expandimg = scaleImage(filtimg, localbin)
                expandimg = frame_constant(expandimg, imgarray.shape)
                filtimg = imgarray - expandimg
        except:
                apDisplay.printWarning("High Pass Filter failed")
                return imgarray
        return filtimg
	def insertClusterRun(self):
		### create a clustering run object
		clusterrunq = appiondata.ApClusteringRunData()
		clusterrunq['runname'] = self.params['runname']
		clusterrunq['pixelsize'] = self.params['apix']
		clusterrunq['boxsize'] = self.params['boxsize']
		clusterrunq['description'] = self.params['description']
		clusterrunq['num_particles'] = self.params['num_particles']
		clusterrunq['alignstack'] = self.analysisdata['alignstack']
		clusterrunq['analysisrun'] = self.analysisdata

		apDisplay.printMsg("inserting clustering parameters into database")
		if self.params['commit'] is True:
			clusterrunq.insert()
		else:
			apDisplay.printWarning("not committing results to DB")
		self.clusterrun = clusterrunq

		return
def weightedLeastSquares(X, Y, W):
    """
        solve using the normal equations with no manipulation
        """
    t0 = time.time()
    ### check the input
    if checkMatrixSizes(X, Y) is False:
        return None
    ### solve it
    XTW = numpy.transpose(X) * W
    XTWX = numpy.dot(XTW, X)
    if numpy.linalg.det(XTWX) == 0:
        apDisplay.printWarning("Singular matrix in calculation")
        return None
    XTWXinv = numpy.linalg.inv(XTWX)
    beta = numpy.dot(numpy.dot(XTWXinv, XTW), Y)
    #apDisplay.printMsg("weightedLeastSquares completed in %s"
    #       %(apDisplay.timeString(time.time()-t0)))
    return beta
	def convertDefociToConvention(self, ctfdata):
		ctfdb.printCtfData(ctfdata)
		initdefocusratio = ctfdata['defocus2']/ctfdata['defocus1']

		# program specific corrections?
		self.angle = ctfdata['angle_astigmatism']
		#angle = round(self.angle/2.5,0)*2.5

		#by convention: abs(ctfdata['defocus1']) < abs(ctfdata['defocus2'])
		if abs(ctfdata['defocus1']) > abs(ctfdata['defocus2']):
			# incorrect, need to shift angle by 90 degrees
			apDisplay.printWarning("|def1| > |def2|, flipping defocus axes")
			self.defocus1 = ctfdata['defocus2']
			self.defocus2 = ctfdata['defocus1']
			self.angle += 90
		else:
			# correct, ratio > 1
			self.defocus1 = ctfdata['defocus1']
			self.defocus2 = ctfdata['defocus2']
		if self.defocus1 < 0 and self.defocus2 < 0:
			apDisplay.printWarning("Negative defocus values, taking absolute value")
			self.defocus1 = abs(self.defocus1)
			self.defocus2 = abs(self.defocus2)
		self.defdiff = self.defocus1 - self.defocus2
		#elliptical ratio is ratio of zero locations NOT defocii
		self.defocusratio = self.defocus2/self.defocus1
		self.ellipratio = ctftools.defocusRatioToEllipseRatio(self.defocus1, self.defocus2, 
			self.initfreq, self.cs, self.volts, self.ampcontrast)

		# get angle within range -90 < angle <= 90
		while self.angle > 90:
			self.angle -= 180
		while self.angle < -90:
			self.angle += 180

		apDisplay.printColor("Final params: def1: %.2e | def2: %.2e | angle: %.1f | defratio %.2f"%
			(self.defocus1, self.defocus2, self.angle, self.defocusratio), "cyan")

		perdiff = abs(self.defocus1-self.defocus2)/abs(self.defocus1+self.defocus2)
		apDisplay.printMsg("Defocus Astig Percent Diff %.2f -- %.3e, %.3e"
				%(perdiff*100,self.defocus1,self.defocus2))

		return
    def start(self):
        self.setFileName()

        scale = float(self.params['oldapix']) / self.params['newapix']

        mrcname = os.path.join(self.params['rundir'],
                               self.params['name'] + ".mrc")
        origmodel = self.params['file']
        if os.path.isfile(mrcname):
            apDisplay.printError("File exists")

        if (abs(self.params['oldapix'] - self.params['newapix']) > 1.0e-2 or
                abs(self.params['oldbox'] - self.params['newbox']) > 1.0e-1):
            ### rescale old model to a new size
            apDisplay.printWarning("rescaling original model to a new size")
            scale = float(self.params['oldapix']) / self.params['newapix']
            apDisplay.printMsg("rescaling model by scale factor of %.4f" %
                               (scale))
            apVolume.rescaleVolume(origmodel, mrcname, self.params['oldapix'],
                                   self.params['newapix'],
                                   self.params['newbox'])
        else:
            ### simple upload, just copy file to models folder
            apDisplay.printMsg("copying original model to a new location: " +
                               mrcname)
            shutil.copyfile(origmodel, mrcname)

        if self.params['viper2eman'] is True:
            apVolume.viper2eman(mrcname, mrcname, apix=self.params['newapix'])

        ### render chimera images of model
        contour = self.params['contour']
        if self.params['mass'] is not None:
            apChimera.setVolumeMass(mrcname, self.params['newapix'],
                                    self.params['mass'])
            contour = 1.0
        apChimera.renderSnapshots(mrcname,
                                  contour=contour,
                                  zoom=self.params['zoom'],
                                  sym=self.params['symdata']['eman_name'])

        self.insertModel(mrcname)
Esempio n. 60
0
def makeFilesForETomoSampleRecon(processdir,
                                 stackdir,
                                 aligndir,
                                 templatedir,
                                 seriesname,
                                 thickness,
                                 pixelsize,
                                 yspacing,
                                 has_rotation=False):
    '''
        Make or link local files required by etomo to redo sampling, creating tomopitch model, and reconstruct the volume.
        '''
    # etomo status file
    createETomoBoundaryModelEDF(processdir, templatedir, seriesname, thickness,
                                pixelsize)
    # required by remaking sample tomograms inside etomo
    if has_rotation:
        apDisplay.printWarning(
            'eTomo wants to regenerate global alignment from non-rotated local alignment.  This alignment with rotation will not work right'
        )
    else:
        # prexf file is generated from database values if this function is called from tomomaker.  It is better not to change it if exists.
        prexf = seriesname + ".prexf"
        alignprexf = os.path.join(aligndir, prexf)
        localprexf = os.path.join(processdir, prexf)
        apFile.safeCopy(alignprexf, localprexf)
        writeETomoNewstComTemplate(processdir, seriesname)
        rawtltname = '%s.rawtlt' % (seriesname)
        shutil.copy(os.path.join(stackdir, rawtltname),
                    os.path.join(processdir, rawtltname))
    # required by tomopitch model making
    tomopitchname = 'tomopitch.com'
    imodcomdir = os.path.join(os.environ['IMOD_DIR'], 'com')
    tomopitchpath = os.path.join(processdir, tomopitchname)
    shutil.copy(os.path.join(imodcomdir, tomopitchname), tomopitchpath)
    apFile.replaceUniqueLinePatternInTxtFile(tomopitchpath, 'SpacingInY',
                                             'SpacingInY\t%.1f\n' % yspacing)

    # required by "Final Aligned Stack" step
    stackname = '%s.st' % (seriesname)
    apFile.safeSymLink(os.path.join(stackdir, stackname),
                       os.path.join(processdir, stackname))