def setupParticles(self):
        self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed")

        self.stack = {}
        self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"])
        self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
        self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"])
        self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"])
        self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"])

        boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2
        apix = self.stack["apix"] * self.params["bin"]

        proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"])
        if self.params["bin"] > 1:
            clipsize = boxsize * self.params["bin"]
            proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize)
        if self.params["highpass"] > 1:
            proccmd += " hp=" + str(self.params["highpass"])
        if self.params["lowpass"] > 1:
            proccmd += " lp=" + str(self.params["lowpass"])
        proccmd += " last=" + str(self.params["numpart"] - 1)
        apFile.removeStack(self.params["localstack"], warn=False)
        apEMAN.executeEmanCmd(proccmd, verbose=True)

        ### convert stack into single spider files
        self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"])
        return (boxsize, apix)
def stackToSpiderStack(stack,spiderstack,apix,boxsize,lp=0,hp=0,bin=1,numpart=0):
	"""
	convert an input stack (i.e. imagic) and write it out to a spider-formatted stack
	"""
	emancmd  = "proc2d "
	if not os.path.isfile(stack):
		apDisplay.printError("stackfile does not exist: "+stack)
	emancmd += stack+" "


	emancmd += "apix="+str(apix)+" "
	if lp > 0:
		emancmd += "lp="+str(lp)+" "
	if hp > 0:
		emancmd += "hp="+str(hp)+" "
	if bin > 1:
		clipboxsize = boxsize*bin
		emancmd += "shrink="+str(bin)+" "
		emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" "
	if numpart > 0:
		emancmd += "last="+str(numpart-1)+" "
	emancmd += "spiderswap edgenorm"
	starttime = time.time()
	apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
	from appionlib import apEMAN
	apEMAN.executeEmanCmd(emancmd, verbose=True)
	apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return
Beispiel #3
0
    def start(self):
        ### copy stack to rundir
        newstack = os.path.join(self.params['rundir'], "start.hed")
        if os.path.isfile(newstack):
            apDisplay.printError("Stack already exists")
        emancmd = "proc2d %s %s" % (self.params['stackfile'], newstack)
        if self.params['normalize'] is True:
            emancmd += " edgenorm"
        apEMAN.executeEmanCmd(emancmd)

        ### set final parameters
        boxsize = apFile.getBoxSize(newstack)
        apDisplay.printMsg("Boxsize: %i" % boxsize[0])
        if not boxsize or boxsize <= 0:
            apDisplay.printError("Could not determine stack size")
        else:
            self.boxsize = boxsize[0]
        self.numpart = apFile.numImagesInStack(newstack)
        apDisplay.printMsg("Num part: %i" % self.numpart)
        if not self.numpart or self.numpart <= 0:
            apDisplay.printError("Could not determine number of particles")
        if self.numpart <= 1:
            apDisplay.printError("Not enough particles to upload")

        apStack.averageStack(newstack)

        #self.params['commit'] = False
        self.createStackData()
Beispiel #4
0
    def prepareStack(self):
        ### setup box and mask sizes
        self.alignstackdata = self.getAlignedStack()
        maskpixrad = self.params['maskrad'] / self.alignstackdata[
            'pixelsize'] / self.params['bin']
        boxpixdiam = int(math.ceil(maskpixrad) + 1) * 2
        if boxpixdiam * self.params['bin'] > self.alignstackdata['boxsize']:
            boxpixdiam = math.floor(self.alignstackdata['boxsize'] /
                                    self.params['bin'])
        clippixdiam = boxpixdiam * self.params['bin']
        apDisplay.printMsg("Pixel mask radius=" + str(maskpixrad))

        ### convert aligned stack to local dir
        oldalignedstack = os.path.join(self.alignstackdata['path']['path'],
                                       self.alignstackdata['imagicfile'])
        alignedstackname = "alignedstack.hed"
        alignedstack = os.path.join(self.params['rundir'], alignedstackname)
        apFile.removeFile(alignedstack)
        emancmd = ("proc2d %s %s shrink=%d clip=%d,%d" %
                   (oldalignedstack, alignedstack, self.params['bin'],
                    clippixdiam, clippixdiam))
        if self.params['numpart'] is not None and self.params[
                'numpart'] < self.numpart:
            emancmd += " last=%d" % (self.params['numpart'] - 1)
            self.numpart = self.params['numpart']
        apEMAN.executeEmanCmd(emancmd, verbose=True)
        return alignedstack
	def createTemplateStack(self):
		"""
		takes the spider file and creates an average template of all particles
		"""

		templatestack = os.path.join(self.params['rundir'], "templatestack00.spi")
		apFile.removeFile(templatestack, warn=True)

		### hack to use standard filtering library
		templateparams = {}
		templateparams['apix'] = self.stack['apix']
		templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
		templateparams['templateIds'] = self.templatelist
		templateparams['bin'] = self.params['bin']
		templateparams['lowpass'] = self.params['lowpass']
		templateparams['median'] = None
		templateparams['pixlimit'] = None
		print templateparams
		apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
		filelist = apTemplate.getTemplates(templateparams)

		for mrcfile in filelist:
			emancmd  = ("proc2d templates/"+mrcfile+" "+templatestack
				+" clip="+str(self.boxsize)+","+str(self.boxsize)
				+" spider ")
			if self.params['inverttemplates'] is True:
				emancmd += " invert "
			apEMAN.executeEmanCmd(emancmd, showcmd=False)

		return templatestack
def boxMaskStack(bmstackf, partdatas, box, xmask, ymask, falloff, imask=None, norotate=False):
        from appionlib.apSpider import operations
        from appionlib import apEMAN
        import os

        # create blank image for mask using SPIDER
        maskfile = "boxmask.spi"
        operations.createBoxMask(maskfile,box,xmask,ymask,falloff,imask)

        # convert mask to MRC
        apEMAN.executeEmanCmd("proc2d boxmask.spi boxmask.mrc",verbose=False,showcmd=False)
        os.remove("boxmask.spi")

        maskarray = mrc.read("boxmask.mrc")

        # box particles
        maskedparts = []
        for i in range(len(partdatas)):
                if norotate is True:
                        rotatemask = maskarray
                else:
                        angle = (-partdatas[i]['angle'])-90
                        rotatemask = ndimage.rotate(maskarray, angle=angle, reshape=False, order=1)
                maskedparts.append(rotatemask)

        # write to stack
        apImagicFile.writeImagic(maskedparts, bmstackf)
        os.remove("boxmask.mrc")
        return bmstackf
	def phaseFlipWholeImage(self, inimgpath, imgdata):
		outimgpath = os.path.join(self.params['rundir'], self.shortname+"-ctfcorrect.dwn.mrc")

		### High tension on CM is given in kv instead of v so do not divide by 1000 in that case
		if imgdata['scope']['tem']['name'] == "CM":
			voltage = imgdata['scope']['high tension']
		else:
			voltage = (imgdata['scope']['high tension'])/1000

		apix = apDatabase.getPixelSize(imgdata)
		defocus, ampconst = self.getDefocusAmpConstForImage(imgdata, True)
		defocus *= 1.0e6
		self.checkDefocus(defocus, self.shortname)
		### get all CTF parameters, we also need to get the CS value from the database
		ctfdata = self.getBestCtfValue(imgdata, False)
		#ampconst = ctfdata['amplitude_contrast'] ### we could use this too

		# find cs
		cs = self.getCS(ctfdata)

		parmstr = ("parm=%f,200,1,%.3f,0,17.4,9,1.53,%i,%.1f,%f" %(defocus, ampconst, voltage, cs, apix))
		emancmd = ("applyctf %s %s %s setparm flipphase" % (inimgpath, outimgpath, parmstr))

		apDisplay.printMsg("phaseflipping entire micrograph with defocus "+str(round(defocus,3))+" microns")
		apEMAN.executeEmanCmd(emancmd, showcmd = True)
		return outimgpath
Beispiel #8
0
    def processFrealignVolume(self, iternum, symname):
        volumeImagicFile = os.path.join(self.params['rundir'],
                                        "threed.%03da.hed" % (iternum))
        if not os.path.isfile(volumeImagicFile):
            apDisplay.printError("Failed to find volume: %s" %
                                 (volumeImagicFile))
        volumeMrcFile = os.path.join(self.params['rundir'],
                                     "threed.%03da.mrc" % (iternum))
        if not os.path.isfile(volumeMrcFile):
            emancmd = "proc3d %s %s apix=%.3f origin=0,0,0 norm=0,1" % (
                volumeImagicFile, volumeMrcFile, self.apix)
            apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
        if not os.path.isfile(volumeMrcFile):
            apDisplay.printError("Failed to convert volume")

        apChimera.filterAndChimera(volumeMrcFile,
                                   res=10,
                                   apix=self.apix,
                                   chimtype='snapshot',
                                   contour=1.0,
                                   zoom=self.params['zoom'],
                                   sym=symname,
                                   silhouette=True,
                                   mass=self.params['mass'])

        return volumeMrcFile
Beispiel #9
0
    def convertSpiderStack(self, spiderstack):
        """
		takes the stack file and creates a spider file ready for processing
		"""
        emancmd = "proc2d "
        if not os.path.isfile(spiderstack):
            apDisplay.printError("stackfile does not exist: " + spiderstack)
        emancmd += spiderstack + " "

        imagicstack = os.path.join(self.params['rundir'], "alignstack.hed")
        apFile.removeFile(imagicstack, warn=True)
        emancmd += imagicstack + " "

        starttime = time.time()
        apDisplay.printColor(
            "Running spider stack conversion this can take a while", "cyan")
        apEMAN.executeEmanCmd(emancmd, verbose=True)
        apDisplay.printColor(
            "finished eman in " +
            apDisplay.timeString(time.time() - starttime), "cyan")

        numpart = apFile.numImagesInStack(imagicstack)
        if numpart != self.params['numpart']:
            apDisplay.printError(
                "The number of particles returned is different from the number requested"
            )

        return imagicstack
	def setupVolumes(self, boxsize, apix):
		voldocfile = "volumelist"+self.timestamp+".doc"
		f = open(voldocfile, "w")
		if self.params['modelstr'] is not None:
			for i, modelid in enumerate(self.params['modelids']):
				### Scale volume
				mrcvolfile = os.path.join(self.params['rundir'], "volume%s_%02d_%05d.mrc"%(self.timestamp, i+1, modelid))
				apModel.rescaleModel(modelid, mrcvolfile, boxsize, apix)

				### Convert volume to spider
				spivolfile = os.path.join(self.params['rundir'], "volume%s_%02d_%05d.spi"%(self.timestamp, i+1, modelid))
				emancmd = "proc3d %s %s spidersingle"%(mrcvolfile, spivolfile)
				apEMAN.executeEmanCmd(emancmd, verbose=False)

				### Normalize volume
				normalvolfile = self.normalizeVolume(spivolfile)

				### Write to selection file
				f.write(normalvolfile+" 1\n")
		else:
			for i in range(self.params['nvol']):
				### Create Gaussian sphere
				spivolfile = os.path.join(self.params['rundir'], "volume%s_%02d.spi"%(self.timestamp, i+1))
				self.createGaussianSphere(spivolfile, boxsize)

				### Normalize volume
				normalvolfile = self.normalizeVolume(spivolfile)

				### Write to selection file
				f.write(normalvolfile+" 1\n")
		f.close()
		return voldocfile
	def genProj(self, euler, density, tilt=None):
		"""
		Rotate model and then project at two tilt angles

		This could go faster if we could add Euler angles...

		90,-90 puts cameras where the would be in the microscope
		"""
		(alt, az, phi) = euler
		rotcmd = ( "proc3d "+density+" rotated.mrc rot="
			+str(alt)+","+str(az)+","+str(phi) )
		apEMAN.executeEmanCmd(rotcmd, verbose=False, showcmd=False)
		if abs(tilt - self.tiltangle) < 5.0:
			bigtilt = self.tiltangle - self.notangle
			euler = ("%.3f,%.3f,%.3f"%(
				bigtilt + random.gauss(0, self.params['slopcam']),
				90+random.gauss(0, self.params['slopcam']),
				-90+random.gauss(0, self.params['slopcam'])))
			cmd = "project3d rotated.mrc out="+self.params['tiltstack']+" euler="+euler
		elif abs(tilt - self.notangle) < 5.0:
			#euler = "%.3f,90,-90"%(0.0)
			euler = ( "%.3f,%.3f,%.3f"%(
				random.gauss(0, self.params['slopcam']),
				90+random.gauss(0, self.params['slopcam']),
				-90+random.gauss(0, self.params['slopcam'])))
			cmd = "project3d rotated.mrc out="+self.params['notstack']+" euler="+euler
		else:
			apDisplay.printError("unknown tilt angle: %.3f"%(tilt))
		apEMAN.executeEmanCmd(cmd, verbose=False, showcmd=False)
    def genProj(self, euler, density, tilt=None):
        """
                Rotate model and then project at two tilt angles

                This could go faster if we could add Euler angles...

                90,-90 puts cameras where the would be in the microscope
                """
        (alt, az, phi) = euler
        rotcmd = ("proc3d " + density + " rotated.mrc rot=" + str(alt) + "," +
                  str(az) + "," + str(phi))
        apEMAN.executeEmanCmd(rotcmd, verbose=False, showcmd=False)
        if abs(tilt - self.tiltangle) < 5.0:
            bigtilt = self.tiltangle - self.notangle
            euler = ("%.3f,%.3f,%.3f" %
                     (bigtilt + random.gauss(0, self.params['slopcam']),
                      90 + random.gauss(0, self.params['slopcam']),
                      -90 + random.gauss(0, self.params['slopcam'])))
            cmd = "project3d rotated.mrc out=" + self.params[
                'tiltstack'] + " euler=" + euler
        elif abs(tilt - self.notangle) < 5.0:
            #euler = "%.3f,90,-90"%(0.0)
            euler = ("%.3f,%.3f,%.3f" %
                     (random.gauss(0, self.params['slopcam']),
                      90 + random.gauss(0, self.params['slopcam']),
                      -90 + random.gauss(0, self.params['slopcam'])))
            cmd = "project3d rotated.mrc out=" + self.params[
                'notstack'] + " euler=" + euler
        else:
            apDisplay.printError("unknown tilt angle: %.3f" % (tilt))
        apEMAN.executeEmanCmd(cmd, verbose=False, showcmd=False)
	def start(self):
		### load parameters
		runparams = self.readRunParameters()

		### align references
		self.alignReferences(runparams)

		### create an aligned stack
		self.createAlignedReferenceStack()

		### read particles
		self.lastiter = self.findLastIterNumber()
		if self.params['sort'] is True:
			self.sortFolder()
		reflist = self.readRefDocFile()
		partlist = self.readPartDocFile(reflist)
		self.writePartDocFile(partlist)

		### create aligned stacks
		alignimagicfile = self.createAlignedStacks(partlist, runparams['localstack'])
		apStack.averageStack(alignimagicfile)

		### calculate resolution for each reference
		apix = apStack.getStackPixelSizeFromStackId(runparams['stackid'])*runparams['bin']
		self.calcResolution(partlist, alignimagicfile, apix)

		### insert into databse
		self.insertRunIntoDatabase(alignimagicfile, runparams)
		self.insertParticlesIntoDatabase(runparams['stackid'], partlist)

		apFile.removeStack(runparams['localstack'], warn=False)
		rmcmd = "/bin/rm -fr partfiles/*"
		apEMAN.executeEmanCmd(rmcmd, verbose=False, showcmd=False)
		apFile.removeFilePattern("partfiles/*")
	def createSpiderFile(self):
		"""
		takes the stack file and creates a spider file ready for processing
		"""
		emancmd  = "proc2d "
		if not os.path.isfile(self.stack['file']):
			apDisplay.printError("stackfile does not exist: "+self.stack['file'])
		emancmd += self.stack['file']+" "

		spiderstack = os.path.join(self.params['rundir'], "start.spi")
		apFile.removeFile(spiderstack, warn=True)
		emancmd += spiderstack+" "

		emancmd += "apix="+str(self.stack['apix'])+" "
		if self.params['lowpass'] > 0:
			emancmd += "lp="+str(self.params['lowpass'])+" "
		if self.params['highpass'] > 0:
			emancmd += "hp="+str(self.params['highpass'])+" "
		if self.params['bin'] > 1:
			clipboxsize = self.boxsize*self.params['bin']
			emancmd += "shrink="+str(self.params['bin'])+" "
			emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" "
		emancmd += "last="+str(self.params['numpart']-1)+" "
		emancmd += "spider edgenorm"
		starttime = time.time()
		apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
		apEMAN.executeEmanCmd(emancmd, verbose=True)
		apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
		return spiderstack
    def createTemplateStack(self):
        """
		takes the spider file and creates an average template of all particles
		"""

        templatestack = os.path.join(self.params['rundir'],
                                     "templatestack00.spi")
        apFile.removeFile(templatestack, warn=True)

        ### hack to use standard filtering library
        templateparams = {}
        templateparams['apix'] = self.stack['apix']
        templateparams['rundir'] = os.path.join(self.params['rundir'],
                                                "templates")
        templateparams['templateIds'] = self.templatelist
        templateparams['bin'] = self.params['bin']
        templateparams['lowpass'] = self.params['lowpass']
        templateparams['median'] = None
        templateparams['pixlimit'] = None
        print templateparams
        apParam.createDirectory(
            os.path.join(self.params['rundir'], "templates"))
        filelist = apTemplate.getTemplates(templateparams)

        for mrcfile in filelist:
            emancmd = ("proc2d templates/" + mrcfile + " " + templatestack +
                       " clip=" + str(self.boxsize) + "," + str(self.boxsize) +
                       " spiderswap ")
            if self.params['inverttemplates'] is True:
                emancmd += " invert "
            apEMAN.executeEmanCmd(emancmd, showcmd=False)

        return templatestack
Beispiel #16
0
	def runEMANmra(self):
		# set up cls files
		emancmd = "proc2d %s.hed cls mraprep" %self.params['currentcls']
		apEMAN.executeEmanCmd(emancmd, verbose=False)

		# run EMAN projection matching
		emancmd = "classesbymra %s %s.hed mask=%i split imask=-1 logit=1 sep=1 phase" %(
			self.params['localstack'],
			self.params['currentcls'],
			self.workingmask
		)
		if self.params['nproc'] > 1:
			apEMAN.executeRunpar(emancmd, self.params['nproc'])
		else:
			apEMAN.executeEmanCmd(emancmd, verbose=True)

		# create stack of aligned particles
		# first get list of cls files
		clslist=glob.glob('cls*.lst')
		clslist.sort()
		emantar = tarfile.open("cls.eman.tar","w")
		clsarray = [[]for i in range(self.params['numpart'])]
		for cls in clslist:
			f = open(cls)
			lines = f.readlines()
			f.close()
			for l in range(1,len(lines)):
				d=lines[l].strip().split()
				if len(d) < 4:
					continue
				part = int(d[0])
				stack = d[1]
				cc = float(d[2][:-1])
				(rot,x,y,mirror) = d[3].split(',')
				clsarray[part]=[part,stack,cc,float(rot),float(x),float(y),int(mirror)]
			# for tarring cls files
			emantar.add(cls)
		emantar.close()

		# remove eman cls####.lst files
		for cls in clslist:
			os.remove(cls)

		# create a new cls file with all particles
		f = open("cls_all.lst","w")
		f.write("#LST\n")
		for p in clsarray:
			f.write("%i\t%s\t%.2f,  %.6f,%.3f,%.3f,%i\n" % (p[0],p[1],p[2],p[3],p[4],p[5],p[6]))
		f.close()

		# create aligned particles
		apEMAN.alignParticlesInLST("cls_all.lst","mrastack.hed")

		if self.params['currentiter'] > 1:
			apFile.removeStack(self.params['alignedstack'])
		self.params['alignedstack']=os.path.abspath("mrastack")
		self.params['alignedstack'] = string.replace(self.params['alignedstack'],"/jetstor/APPION","")

		# remove cls file used for alignment
		os.remove("cls_all.lst")
Beispiel #17
0
    def processTemplate(self, mrcfile):
        ### shrink
        apDisplay.printMsg("Binning template by a factor of " +
                           str(self.params['bin']))
        clipsize = int(
            math.floor(self.stack['boxsize'] / self.params['bin']) *
            self.params['bin'])
        emancmd = ("proc2d " + mrcfile + " " + mrcfile + " shrink=" +
                   str(self.params['bin']) + " spiderswap ")
        emancmd += "clip=" + str(clipsize) + "," + str(clipsize) + " "
        apEMAN.executeEmanCmd(emancmd)

        ### normalize and center
        #apDisplay.printMsg("Normalize and centering template")
        #emancmd = "proc2d "+mrcfile+" "+mrcfile+" edgenorm center"
        #apEMAN.executeEmanCmd(emancmd)

        ### convert to SPIDER
        apDisplay.printMsg("Converting template to SPIDER")
        templatefile = "template.spi"
        if os.path.isfile(templatefile):
            apFile.removeFile(templatefile, warn=True)
        emancmd = "proc2d template.mrc " + templatefile + " spiderswap "
        apEMAN.executeEmanCmd(emancmd)

        return templatefile
Beispiel #18
0
    def selectRandomParticles(self):
        """
		takes the spider file and creates an average template of all particles
		"""
        ### create random keep list
        numrandpart = int(self.params['numpart'] / 100) + 2
        apDisplay.printMsg("Selecting 1% of particles (" + str(numrandpart) +
                           ") to average")
        # create random list
        keepdict = {}
        randlist = []
        for i in range(numrandpart):
            rand = int(random.random() * self.params['numpart'])
            while rand in keepdict:
                rand = int(random.random() * self.params['numpart'])
            keepdict[rand] = 1
            randlist.append(rand)
        # sort and write to file
        randlist.sort()
        f = open("randkeep.lst", "w")
        for rand in randlist:
            f.write(str(rand) + "\n")
        f.close()

        emancmd = "proc2d " + self.stack[
            'file'] + " template.mrc list=randkeep.lst average edgenorm"
        apEMAN.executeEmanCmd(emancmd)

        templatefile = self.processTemplate("template.mrc")

        return templatefile
	def convertStackToSpider(self, emanstackfile, classnum):
		"""
		takes the stack file and creates a spider file ready for processing
		"""
		if not os.path.isfile(emanstackfile):
			apDisplay.printError("stackfile does not exist: "+emanstackfile)

		### first high pass filter particles
		apDisplay.printMsg("pre-filtering particles")
		apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
		emancmd = ("proc2d "+emanstackfile+" "+emanstackfile
			+" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
			+" inplace")
		apEMAN.executeEmanCmd(emancmd, verbose=True)

		### convert imagic stack to spider
		emancmd  = "proc2d "
		emancmd += emanstackfile+" "
		spiderstack = os.path.join(self.params['rundir'], str(classnum), "otrstack"+self.timestamp+".spi")
		apFile.removeFile(spiderstack, warn=True)
		emancmd += spiderstack+" "

		emancmd += "spiderswap edgenorm"
		starttime = time.time()
		apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
		apEMAN.executeEmanCmd(emancmd, verbose=True)
		apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
		return spiderstack
Beispiel #20
0
        def useClusterForTemplate(self):
                apDisplay.printMsg("Using clustering stack to make templates")
                clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
                self.params['apix'] = clusterstackdata['clusterrun']['alignstack']['pixelsize']
                stackfile = os.path.join(clusterstackdata['path']['path'], clusterstackdata['avg_imagicfile'])
                self.params['templatelist'] = []
                stackid = clusterstackdata['clusterrun']['alignstack']['stack'].dbid
                sessiondata = apStack.getSessionDataFromStackId(stackid)
                self.params['sessionname'] = sessiondata['name']

                ### check to see if stackimagenum is within the boundary of the stack
                numpart = apFile.numImagesInStack(stackfile)
                for i in self.params['imgnums'].split(","):
                        partnum = int(i)
                        if partnum > numpart or partnum < 0:
                                apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart))
                        apDisplay.printMsg("Extracting image %d from cluster stack" % partnum)
                        templatename = "cluster%d-average%d.mrc"%(self.params['clusterid'],partnum)
                        abstemplatepath= os.path.join(clusterstackdata['path']['path'], templatename)
                        ### run proc2d with params['stackimgnum']
                        emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum)
                        ###  create template
                        apDisplay.printMsg("creating "+templatename)
                        apEMAN.executeEmanCmd(emancmd)
                        if os.path.isfile(abstemplatepath):
                                self.params['templatelist'].append(abstemplatepath)
	def setupParticles(self):
		self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")

		self.stack = {}
		self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
		self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
		self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
		self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
		self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])

		boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
		apix = self.stack['apix']*self.params['bin']

		proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
		if self.params['bin'] > 1:
			clipsize = boxsize*self.params['bin']
			proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
		if self.params['highpass'] > 1:
			proccmd += " hp="+str(self.params['highpass'])
		if self.params['lowpass'] > 1:
			proccmd += " lp="+str(self.params['lowpass'])
		proccmd += " last="+str(self.params['numpart']-1)
		apFile.removeStack(self.params['localstack'], warn=False)
		apEMAN.executeEmanCmd(proccmd, verbose=True)

		### convert stack into single spider files
		self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
		return (boxsize, apix)
def rescaleVolume(infile, outfile, inapix, outapix=None, newbox=None, spider=False):
	"""
	scale an existing model - provide an input model & output (strings)
	an input a/pix & output a/pix (floats)
	and the final box size (after scaling)
	currently uses EMAN's proc3d to do this
	"""

	origbox = getModelDimensions(infile)
	if newbox is None:
		newbox = origbox

	if outapix is None:
		outapix = inapix

	apDisplay.printMsg( ("rescaling %s with boxsize %d by a factor of %.3f\n"
		+"\tand saving to %s with a boxsize %d")
		%(infile, origbox, scalefactor, outfile, newbox))

	emancmd = "proc3d %s %s " % (infile, outfile)

	scalefactor = float(inapix/outapix)
	# only do it if it is bigger than 10%
	if abs(scalefactor-1.0) > 0.1:
		emancmd += "scale=%.3f " % scalefactor

	emancmd += "clip=%i,%i,%i norm=0,1 " % (newbox, newbox, newbox)
	apEMAN.executeEmanCmd(emancmd, verbose=True)

	if spider is True:
		emancmd += "spidersingle "
	return
    def createSpiderFile(self):
        """
                takes the stack file and creates a spider file ready for processing
                """
        emancmd = "proc2d "
        if not os.path.isfile(self.stack["file"]):
            apDisplay.printError("stackfile does not exist: " + self.stack["file"])
        emancmd += self.stack["file"] + " "

        spiderstack = os.path.join(self.params["rundir"], "start.spi")
        apFile.removeFile(spiderstack, warn=True)
        emancmd += spiderstack + " "

        emancmd += "apix=" + str(self.stack["apix"]) + " "
        if self.params["lowpass"] > 0:
            emancmd += "lp=" + str(self.params["lowpass"]) + " "
        emancmd += "last=" + str(self.params["numpart"] - 1) + " "
        emancmd += "shrink=" + str(self.params["bin"]) + " "
        clipsize = int(math.floor(self.stack["boxsize"] / self.params["bin"] / 2.0) * self.params["bin"] * 2)
        emancmd += "clip=" + str(clipsize) + "," + str(clipsize) + " "
        emancmd += "spiderswap edgenorm"
        starttime = time.time()
        apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
        apEMAN.executeEmanCmd(emancmd, verbose=True)
        apDisplay.printColor("finished eman in " + apDisplay.timeString(time.time() - starttime), "cyan")
        return spiderstack
def MRCtoSPI(infile, rundir):
	# convert file to spider file
	tmpspifile = apParam.randomString(8)+".spi"
	tmpspifile = os.path.join(rundir,tmpspifile)
	emancmd = "proc3d %s %s spidersingle" %(infile,tmpspifile)
	apEMAN.executeEmanCmd(emancmd, verbose=True)
	return tmpspifile
        def createOrientationReference(self):
                """
                convert spider orientation reference
                """

                orientref = os.path.join(self.params['rundir'], "orient.spi")
                apFile.removeFile(orientref, warn=True)

                #templatedata = apTemplate.getTemplateFromId(self.params['orientref'])
                #templatefile = os.path.join(templatedata['path']['path'], templatedata['templatename'])

                ### hack to use standard filtering library
                templateparams = {}
                templateparams['apix'] = self.stack['apix']
                templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
                templateparams['templateIds'] = [self.params['orientref'],]
                templateparams['bin'] = self.params['bin']
                templateparams['lowpass'] = self.params['lowpass']
                templateparams['median'] = None
                templateparams['pixlimit'] = None
                print 'Converting orientation reference:\n', templateparams
                apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
                filelist = apTemplate.getTemplates(templateparams)
                mrcfile = filelist[0]

                localclip = self.clipsize/self.params['bin']
                emancmd  = ("proc2d templates/"+mrcfile+" "+orientref
                        +" clip="+str(localclip)+","+str(localclip)
                        +" edgenorm spiderswap-single ")
                if self.params['inverttemplates'] is True:
                        emancmd += " invert "
                apEMAN.executeEmanCmd(emancmd, showcmd=False)

                return orientref
Beispiel #26
0
    def createSpiderFile(self):
        """
                takes the stack file and creates a spider file ready for processing
                """
        emancmd = "proc2d "
        if not os.path.isfile(self.stack['file']):
            apDisplay.printError("stackfile does not exist: " +
                                 self.stack['file'])
        emancmd += self.stack['file'] + " "

        spiderstack = os.path.join(self.params['rundir'], "start.spi")
        apFile.removeFile(spiderstack, warn=True)
        emancmd += spiderstack + " "

        emancmd += "apix=" + str(self.stack['apix']) + " "
        if self.params['lowpass'] > 0:
            emancmd += "lp=" + str(self.params['lowpass']) + " "
        emancmd += "last=" + str(self.params['numpart'] - 1) + " "
        emancmd += "shrink=" + str(self.params['bin']) + " "
        emancmd += "clip=" + str(self.clipsize) + "," + str(
            self.clipsize) + " "
        emancmd += "spiderswap"
        starttime = time.time()
        apDisplay.printColor("Converting stack to spider format", "cyan")
        apEMAN.executeEmanCmd(emancmd, verbose=True)
        apDisplay.printColor(
            "finished eman in " +
            apDisplay.timeString(time.time() - starttime), "cyan")
        return spiderstack
	def start(self):
		### load parameters
		runparams = self.readRunParameters()

		### align references
		self.alignReferences(runparams)

		### create an aligned stack
		self.createAlignedReferenceStack()

		### read particles
		self.lastiter = self.findLastIterNumber()
		if self.params['sort'] is True:
			self.sortFolder()
		reflist = self.readRefDocFile()
		partlist = self.readPartDocFile(reflist)
		self.writePartDocFile(partlist)

		### create aligned stacks
		alignimagicfile = self.createAlignedStacks(partlist, runparams['localstack'])
		apStack.averageStack(alignimagicfile)

		### calculate resolution for each reference
		apix = apStack.getStackPixelSizeFromStackId(runparams['stackid'])*runparams['bin']
		self.calcResolution(partlist, alignimagicfile, apix)

		### insert into databse
		self.insertRunIntoDatabase(alignimagicfile, runparams)
		self.insertParticlesIntoDatabase(runparams['stackid'], partlist)

		apFile.removeStack(runparams['localstack'], warn=False)
		rmcmd = "/bin/rm -fr partfiles/*"
		apEMAN.executeEmanCmd(rmcmd, verbose=False, showcmd=False)
		apFile.removeFilePattern("partfiles/*")
Beispiel #28
0
    def createTemplateStack(self):
        """
                convert spider template stack
                """

        templatestack = os.path.join(self.params['rundir'],
                                     "templatestack.spi")
        apFile.removeFile(templatestack, warn=True)

        ### hack to use standard filtering library
        templateparams = {}
        templateparams['apix'] = self.stack['apix']
        templateparams['rundir'] = os.path.join(self.params['rundir'],
                                                "templates")
        templateparams['templateIds'] = self.templatelist
        templateparams['bin'] = self.params['bin']
        templateparams['lowpass'] = self.params['lowpass']
        templateparams['median'] = None
        templateparams['pixlimit'] = None
        print 'Converting reference templates:\n', templateparams
        apParam.createDirectory(
            os.path.join(self.params['rundir'], "templates"))
        filelist = apTemplate.getTemplates(templateparams)

        localclip = self.clipsize / self.params['bin']
        for mrcfile in filelist:
            emancmd = ("proc2d templates/" + mrcfile + " " + templatestack +
                       " clip=" + str(localclip) + "," + str(localclip) +
                       " edgenorm spiderswap ")
            if self.params['inverttemplates'] is True:
                emancmd += " invert "
            apEMAN.executeEmanCmd(emancmd, showcmd=False)

        return templatestack
Beispiel #29
0
        def setupVolumes(self, boxsize, apix):
                voldocfile = "volumelist"+self.timestamp+".doc"
                f = open(voldocfile, "w")
                if self.params['modelstr'] is not None:
                        for i, modelid in enumerate(self.params['modelids']):
                                ### Scale volume
                                mrcvolfile = os.path.join(self.params['rundir'], "volume%s_%02d_%05d.mrc"%(self.timestamp, i+1, modelid))
                                apModel.rescaleModel(modelid, mrcvolfile, boxsize, apix)

                                ### Convert volume to spider
                                spivolfile = os.path.join(self.params['rundir'], "volume%s_%02d_%05d.spi"%(self.timestamp, i+1, modelid))
                                emancmd = "proc3d %s %s spidersingle"%(mrcvolfile, spivolfile)
                                apEMAN.executeEmanCmd(emancmd, verbose=False)

                                ### Normalize volume
                                normalvolfile = self.normalizeVolume(spivolfile)

                                ### Write to selection file
                                f.write(normalvolfile+" 1\n")
                else:
                        for i in range(self.params['nvol']):
                                ### Create Gaussian sphere
                                spivolfile = os.path.join(self.params['rundir'], "volume%s_%02d.spi"%(self.timestamp, i+1))
                                self.createGaussianSphere(spivolfile, boxsize)

                                ### Normalize volume
                                normalvolfile = self.normalizeVolume(spivolfile)

                                ### Write to selection file
                                f.write(normalvolfile+" 1\n")
                f.close()
                return voldocfile
def stackToSpiderStack(stack,spiderstack,apix,boxsize,lp=0,hp=0,bin=1,numpart=0):
	"""
	convert an input stack (i.e. imagic) and write it out to a spider-formatted stack
	"""
	emancmd  = "proc2d "
	if not os.path.isfile(stack):
		apDisplay.printError("stackfile does not exist: "+stack)
	emancmd += stack+" "

	apFile.removeFile(spiderstack, warn=True)
	emancmd += spiderstack+" "

	emancmd += "apix="+str(apix)+" "
	if lp > 0:
		emancmd += "lp="+str(lp)+" "
	if hp > 0:
		emancmd += "hp="+str(hp)+" "
	if bin > 1:
		clipboxsize = boxsize*bin
		emancmd += "shrink="+str(bin)+" "
		emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" "
	if numpart > 0:
		emancmd += "last="+str(numpart-1)+" "
	emancmd += "spiderswap edgenorm"
	starttime = time.time()
	apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
	from appionlib import apEMAN
	apEMAN.executeEmanCmd(emancmd, verbose=True)
	apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return
	def start(self):
		### copy stack to rundir
		newstack = os.path.join(self.params['rundir'], "start.hed")
		if os.path.isfile(newstack):
			apDisplay.printError("Stack already exists")
		emancmd = "proc2d %s %s"%(self.params['stackfile'], newstack)
		if self.params['normalize'] is True:
			emancmd += " edgenorm"
		apEMAN.executeEmanCmd(emancmd)

		### set final parameters
		boxsize = apFile.getBoxSize(newstack)
		apDisplay.printMsg("Boxsize: %i"%boxsize[0])
		if not boxsize or boxsize <= 0:
			apDisplay.printError("Could not determine stack size")
		else:
			self.boxsize = boxsize[0]
		self.numpart = apFile.numImagesInStack(newstack)
		apDisplay.printMsg("Num part: %i"%self.numpart)
		if not self.numpart or self.numpart <= 0:
			apDisplay.printError("Could not determine number of particles")
		if self.numpart <= 1:
			apDisplay.printError("Not enough particles to upload")

		apStack.averageStack(newstack)

		#self.params['commit'] = False
		self.createStackData()
def centerParticles(stack, mask=None, maxshift=None):
        apDisplay.printMsg("Centering stack: "+stack)

        stacksize = apFile.stackSize(stack)
        freemem = mem.free()*1024 #convert memory to bytes
        apDisplay.printMsg("file is %s, mem is %s"
                %(apDisplay.bytes(stacksize), apDisplay.bytes(freemem)))
        ### from EMAN FAQ: need to have at least 3x as much ram as the size of the file
        memsize = freemem/3.0
        numfrac = int(math.ceil(stacksize/memsize))

        apDisplay.printMsg("file is %s, will be split into %d fractions"
                %(apDisplay.bytes(stacksize), numfrac))

        for i in range(numfrac):
                emancmd = "cenalignint "+stack
                if numfrac > 1:
                        emancmd += " frac="+str(i)+"/"+str(numfrac)
                if mask is not None:
                        emancmd += " mask="+str(mask)
                if maxshift is not None:
                        emancmd += " maxshift="+str(maxshift)
                apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)

        return
	def start(self):
		"""
		This is the core of your function.
		"""
		stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
		original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
		filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp+".hed")
		apFile.removeStack(filtered_stackfile, warn=False)
		apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
		boxsize = apStack.getStackBoxsize(self.params['stackid'])

		emancmd = "proc2d %s %s apix=%.3f "%(original_stackfile, filtered_stackfile, apix)
		if self.params['lowpass'] is not None:
			emancmd += " lp=%.3f "%(self.params['lowpass'])
		if self.params['highpass'] is not None:
			emancmd += " hp=%.3f "%(self.params['highpass'])
		if self.params['bin'] is not None and self.params['bin'] > 1:
			## determine a multiple of the bin that is divisible by 2 and less than the boxsize
			clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2*self.params['bin']
			emancmd += " shrink=%d clip=%d,%d "%(self.params['bin'], clipsize, clipsize)		
		emancmd += " last=%d "%(self.params['numpart']-1)
		apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)

		### confirm that it worked
		if self.params['numpart'] != apFile.numImagesInStack(filtered_stackfile):
			apDisplay.printError("Missing particles in stack")

		### run the radon transform code
		self.radonAlign(filtered_stackfile)
		
		### insert info into database
		self.commitToDatabase()
def makeNewStack(oldstack, newstack, listfile=None, remove=False, bad=False):
        if not os.path.isfile(oldstack):
                apDisplay.printWarning("could not find old stack: "+oldstack)
        if os.path.isfile(newstack):
                if remove is True:
                        apDisplay.printWarning("removing old stack: "+newstack)
                        time.sleep(2)
                        apFile.removeStack(newstack)
                else:
                        apDisplay.printError("new stack already exists: "+newstack)
        apDisplay.printMsg("creating a new stack\n\t"+newstack+
                "\nfrom the oldstack\n\t"+oldstack+"\nusing list file\n\t"+str(listfile))
        emancmd = "proc2d "+oldstack+" "+newstack
        if listfile is not None:
                emancmd += " list="+listfile
        apEMAN.executeEmanCmd(emancmd, verbose=True)
        if bad is True and listfile is not None:
                ### run only if num bad particles < num good particles
                newstacknumpart = apFile.numImagesInStack(newstack)
                oldstacknumpart = apFile.numImagesInStack(oldstack)
                if newstacknumpart > oldstacknumpart/2:
                        ### create bad.hed stack with all bad particles
                        badstack = os.path.join(os.path.dirname(newstack), "bad.hed")
                        emancmd = "proc2d %s %s exclude=%s"%(oldstack, badstack, listfile)
                        apEMAN.executeEmanCmd(emancmd, verbose=True)
                else:
                        apDisplay.printMsg("Rejecting more particles than keeping, not creating a bad stack")
        return
	def runRmeasure(self, volfile):
		emancmd = "proc3d %s %s"%(volfile, "rmeasure.mrc")
		apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
		apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin']
		self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc")
		#apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan")
		apFile.removeFile("rmeasure.mrc")
def centerParticles(stack, mask=None, maxshift=None):
	apDisplay.printMsg("Centering stack: "+stack)

	stacksize = apFile.stackSize(stack)
	freemem = mem.free()*1024 #convert memory to bytes
	apDisplay.printMsg("file is %s, mem is %s"
		%(apDisplay.bytes(stacksize), apDisplay.bytes(freemem)))
	### from EMAN FAQ: need to have at least 3x as much ram as the size of the file
	memsize = freemem/3.0
	numfrac = int(math.ceil(stacksize/memsize))

	apDisplay.printMsg("file is %s, will be split into %d fractions"
		%(apDisplay.bytes(stacksize), numfrac))

	for i in range(numfrac):
		emancmd = "cenalignint "+stack
		if numfrac > 1:
			emancmd += " frac="+str(i)+"/"+str(numfrac)
		if mask is not None:
			emancmd += " mask="+str(mask)
		if maxshift is not None:
			emancmd += " maxshift="+str(maxshift)
		apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)

	return
def assemblePngs(keys, tag, montagestack):
	apDisplay.printMsg("assembling pngs into montage")
	# get path from montagestack
	stackpath = os.path.dirname(os.path.abspath(montagestack))

	montagecmd = "montage -geometry +4+4 "
	montagestackdata = apImagicFile.readImagic(montagestack)
	for i,key in enumerate(keys):
		if i % 20 == 0:
			sys.stderr.write(".")
		pngfile = key+".png"
		array = montagestackdata['images'][i]
		apImage.arrayToPng(array, pngfile, normalize=True, msg=False)
		#proccmd = "proc2d "+montagestack+" "+pngfile+" first="+str(i)+" last="+str(i)
		#apEMAN.executeEmanCmd(proccmd, verbose=False, showcmd=False)
		montagecmd += pngfile+" "
	apDisplay.printMsg("montaging")
	montagefile = os.path.join(stackpath,"montage"+tag+".png")
	montagecmd += montagefile
	apEMAN.executeEmanCmd(montagecmd, verbose=True)
	#rotatecmd = "mogrify -rotate 180 -flop "+montagefile
	#apEMAN.executeEmanCmd(rotatecmd, verbose=False)
	for key in keys:
		apFile.removeFile(key+".png")
	if not os.path.isfile(montagefile):
		apDisplay.printWarning("failed to create montage file")
	def convertStackToSpider(self, emanstackfile, spiderstack=None):
		"""
		takes the stack file and creates a spider file ready for processing
		"""
		if not os.path.isfile(emanstackfile):
			apDisplay.printError("stackfile does not exist: "+emanstackfile)

		### first high pass filter particles
		#apDisplay.printMsg("pre-filtering particles")
		#apix = apStack.getStackPixelSizeFromStackId(self.params['partstackid'])
		#emancmd = ("proc2d "+emanstackfile+" "+emanstackfile
		#	+" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
		#	+" inplace")
		#apEMAN.executeEmanCmd(emancmd, verbose=True)

		### convert imagic stack to spider
		emancmd  = "proc2d "
		emancmd += emanstackfile+" "
		if spiderstack is None:
			spiderstack = os.path.join(self.params['rundir'], "start.spi")
		apFile.removeFile(spiderstack, warn=True)
		emancmd += spiderstack+" "

		emancmd += "spiderswap edgenorm"
		starttime = time.time()
		apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
		apEMAN.executeEmanCmd(emancmd, verbose=True)
		apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
		return spiderstack
	def useClusterForTemplate(self):
		apDisplay.printMsg("Using clustering stack to make templates")
		clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
		self.params['apix'] = clusterstackdata['clusterrun']['alignstack']['pixelsize']
		stackfile = os.path.join(clusterstackdata['path']['path'], clusterstackdata['avg_imagicfile'])
		self.params['templatelist'] = []
		stackid = clusterstackdata['clusterrun']['alignstack']['stack'].dbid
		sessiondata = apStack.getSessionDataFromStackId(stackid)
		self.params['sessionname'] = sessiondata['name']

		### check to see if stackimagenum is within the boundary of the stack
		numpart = apFile.numImagesInStack(stackfile)
		for i in self.params['imgnums'].split(","):
			partnum = int(i)
			if partnum > numpart or partnum < 0:
				apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart))
			apDisplay.printMsg("Extracting image %d from cluster stack" % partnum)
			templatename = "cluster%d-average%d.mrc"%(self.params['clusterid'],partnum)
			abstemplatepath= os.path.join(clusterstackdata['path']['path'], templatename)
			### run proc2d with params['stackimgnum']
			emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum)
			###  create template
			apDisplay.printMsg("creating "+templatename)
			apEMAN.executeEmanCmd(emancmd)
			if os.path.isfile(abstemplatepath):
				self.params['templatelist'].append(abstemplatepath)
	def runRmeasure(self, volfile):
		emancmd = "proc3d %s %s"%(volfile, "rmeasure.mrc")
		apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
		apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['bin']
		self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc")
		#apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan")
		apFile.removeFile("rmeasure.mrc")
def makeStackInFormatFromDefault(stackdata, format="eman"):
    """
		Function to create the stack in the requested format
		Current default is Imagic format used in EMAN
	"""
    if format not in formats:
        apDisplay.printError("Unknown stack format %s requested" % format)
    stackroot, instackdir, outstackdir = getFormattedStackInfo(stackdata, format)
    instackpath = os.path.join(instackdir, stackroot + ".hed")
    if format == "eman":
        return stackdata["path"]["path"]
    apParam.createDirectory(outstackdir)
    rundir = os.getcwd()
    os.chdir(outstackdir)
    if format == "spider":
        outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0])
        emancmd = "proc2d %s %s spidersingle" % (instackpath, outstackpath)
        apEMAN.executeEmanCmd(emancmd, showcmd=True)
    if format == "frealign":
        outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0])
        emancmd = "proc2d %s %s invert" % (instackpath, outstackpath)
        apEMAN.executeEmanCmd(emancmd, showcmd=True)
    if format == "xmipp":
        ### convert stack into single spider files
        selfile = apXmipp.breakupStackIntoSingleFiles(instackpath)
    os.chdir(rundir)
    return outstackdir
        def createTemplateStack(self):
                """
                convert spider template stack
                """

                templatestack = os.path.join(self.params['rundir'], "templatestack.spi")
                apFile.removeFile(templatestack, warn=True)

                ### hack to use standard filtering library
                templateparams = {}
                templateparams['apix'] = self.stack['apix']
                templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
                templateparams['templateIds'] = self.templatelist
                templateparams['bin'] = self.params['bin']
                templateparams['lowpass'] = self.params['lowpass']
                templateparams['median'] = None
                templateparams['pixlimit'] = None
                print 'Converting reference templates:\n', templateparams
                apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
                filelist = apTemplate.getTemplates(templateparams)

                localclip = self.clipsize/self.params['bin']
                for mrcfile in filelist:
                        emancmd  = ("proc2d templates/"+mrcfile+" "+templatestack
                                +" clip="+str(localclip)+","+str(localclip)
                                +" edgenorm spiderswap ")
                        if self.params['inverttemplates'] is True:
                                emancmd += " invert "
                        apEMAN.executeEmanCmd(emancmd, showcmd=False)

                return templatestack
    def selectRandomParticles(self):
        """
                takes the spider file and creates an average template of all particles
                """
        ### create random keep list
        numrandpart = int(self.params["numpart"] / 100) + 2
        apDisplay.printMsg("Selecting 1% of particles (" + str(numrandpart) + ") to average")
        # create random list
        keepdict = {}
        randlist = []
        for i in range(numrandpart):
            rand = int(random.random() * self.params["numpart"])
            while rand in keepdict:
                rand = int(random.random() * self.params["numpart"])
            keepdict[rand] = 1
            randlist.append(rand)
        # sort and write to file
        randlist.sort()
        f = open("randkeep.lst", "w")
        for rand in randlist:
            f.write(str(rand) + "\n")
        f.close()

        emancmd = "proc2d " + self.stack["file"] + " template.mrc list=randkeep.lst average edgenorm"
        apEMAN.executeEmanCmd(emancmd)

        templatefile = self.processTemplate("template.mrc")

        return templatefile
Beispiel #44
0
        def setupParticles(self):
                self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")

                self.stack = {}
                self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
                self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
                self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
                self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
                self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])

                boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
                apix = self.stack['apix']*self.params['bin']

                proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
                if self.params['bin'] > 1:
                        clipsize = boxsize*self.params['bin']
                        proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
                if self.params['highpass'] > 1:
                        proccmd += " hp="+str(self.params['highpass'])
                if self.params['lowpass'] > 1:
                        proccmd += " lp="+str(self.params['lowpass'])
                proccmd += " last="+str(self.params['numpart']-1)
                apFile.removeStack(self.params['localstack'], warn=False)
                apEMAN.executeEmanCmd(proccmd, verbose=True)

                ### convert stack into single spider files
                self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
                return (boxsize, apix)
	def start(self):
		self.runtime = 0

		self.checkCoranRun()

		### convert stack to spider
		self.alignstackdata = self.getAlignedStack()
		maskpixrad = self.params['maskrad']/self.alignstackdata['pixelsize']/self.params['bin']
		boxpixdiam = int(math.ceil(maskpixrad)+1)*2
		if boxpixdiam*self.params['bin'] > self.alignstackdata['boxsize']:
			boxpixdiam = math.floor(self.alignstackdata['boxsize']/self.params['bin'])
		clippixdiam = boxpixdiam*self.params['bin']
		apDisplay.printMsg("Pixel mask radius="+str(maskpixrad))

		oldalignedstack = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['imagicfile'])
		alignedstackname = re.sub("\.", "_", self.alignstackdata['imagicfile'])+".spi"
		alignedstack = os.path.join(self.params['rundir'], alignedstackname)
		apFile.removeFile(alignedstack)
		emancmd = ("proc2d %s %s spiderswap shrink=%d clip=%d,%d edgenorm"
			%(oldalignedstack,alignedstack,self.params['bin'],clippixdiam,clippixdiam))
		if self.params['numpart'] is not None:
			emancmd += " last=%d"%(self.params['numpart']-1)
			numpart = self.params['numpart']
		else:
			numpart = self.getNumAlignedParticles()
		apEMAN.executeEmanCmd(emancmd, verbose=True)

		### get number of processors
		nproc = apParam.getNumProcessors()

		esttime = classification.estimateTime(numpart, maskpixrad)
		apDisplay.printColor("Running spider this can take awhile, estimated time: "+\
			apDisplay.timeString(esttime),"cyan")

		### do correspondence analysis
		corantime = time.time()
		self.contriblist = classification.correspondenceAnalysis( alignedstack,
			boxsize=boxpixdiam, maskpixrad=maskpixrad,
			numpart=numpart, numfactors=self.params['numfactors'], nproc=nproc)
		corantime = time.time() - corantime

		### make dendrogram
		dendrotime = time.time()
		classification.makeDendrogram(numfactors=min(3,self.params['numfactors']),nproc=nproc)
		dendrotime = time.time() - dendrotime

		inserttime = time.time()
		if self.params['commit'] is True:
			self.runtime = corantime
			self.insertCoranRun(insert=True)
		else:
			apDisplay.printWarning("not committing results to DB")
		inserttime = time.time() - inserttime

		apFile.removeFile(alignedstack, warn=True)

		apDisplay.printMsg("Correspondence Analysis time: "+apDisplay.timeString(corantime))
		apDisplay.printMsg("Make Dendrogram time: "+apDisplay.timeString(dendrotime))
		apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"):
        """
        inputs:
                aligned particle stack
                number of classes
                timestamp
                output directory
        output:
                class averages
                class variances
                dendrogram.png
        """
        if timestamp is None:
                timestamp = apParam.makeTimestamp()

        classavg = rundir+"/"+("classavgstack_%s_%03d" %  (timestamp, numclasses))
        classvar = rundir+"/"+("classvarstack_%s_%03d" %  (timestamp, numclasses))

        thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext)

        ### create class doc files
        mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
        mySpider.toSpider(
                "CL HE",
                thresh,
                spyder.fileFilter(dendrogramfile), # dendrogram doc file
                rundir+"/classdoc_"+timestamp+"_****", # class doc file
        )

        ### delete existing files
        sys.stderr.write("delete existing files")
        for dext in (".hed", ".img", dataext):
                apFile.removeFile(classavg+dext)
                apFile.removeFile(classvar+dext)
        print ""

        ### create class averages
        sys.stderr.write("create class averages")
        for i in range(classes):
                sys.stderr.write(".")
                classnum = i+1
                mySpider.toSpiderQuiet(
                        "AS R",
                        spyder.fileFilter(alignedstack)+"@******",
                        rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)),
                        "A",
                        (classavg+"@%04d" % (classnum)),
                        (classvar+"@%04d" % (classnum)),
                )
        mySpider.close()
        print ""

        ### convert to IMAGIC
        emancmd = "proc2d "+classavg+".spi "+classavg+".hed"
        apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
        emancmd = "proc2d "+classvar+".spi "+classvar+".hed"
        apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)

        return classavg,classvar
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"):
	"""
	inputs:
		aligned particle stack
		number of classes
		timestamp
		output directory
	output:
		class averages
		class variances
		dendrogram.png
	"""
	if timestamp is None:
		timestamp = apParam.makeTimestamp()

	classavg = rundir+"/"+("classavgstack_%s_%03d" %  (timestamp, numclasses))
	classvar = rundir+"/"+("classvarstack_%s_%03d" %  (timestamp, numclasses))

	thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext)

	### create class doc files
	mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
	mySpider.toSpider(
		"CL HE",
		thresh,
		spyder.fileFilter(dendrogramfile), # dendrogram doc file
		rundir+"/classdoc_"+timestamp+"_****", # class doc file
	)

	### delete existing files
	sys.stderr.write("delete existing files")
	for dext in (".hed", ".img", dataext):
		apFile.removeFile(classavg+dext)
		apFile.removeFile(classvar+dext)
	print ""

	### create class averages
	sys.stderr.write("create %i class averages"%classes)
	for i in range(classes):
		sys.stderr.write(".")
		classnum = i+1
		mySpider.toSpiderQuiet(
			"AS R",
			spyder.fileFilter(alignedstack)+"@******",
			rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)),
			"A",
			(classavg+"@%04d" % (classnum)),
			(classvar+"@%04d" % (classnum)),
		)
	mySpider.close()
	print ""

	### convert to IMAGIC
	emancmd = "proc2d "+classavg+".spi "+classavg+".hed"
	apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
	emancmd = "proc2d "+classvar+".spi "+classvar+".hed"
	apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)

	return classavg,classvar
def emanSpiderToStack(partlist):
        apFile.removeStack("emanspi.hed", warn=False)
        for part in partlist:
                spider.write(part, "temp.spi")
                emancmd = "proc2d temp.spi emanspi.hed"
                apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
                apFile.removeFile("temp.spi")
        return
	def runRmeasure(self):
		finalrawvolfile = os.path.join(self.params['rundir'], "rawvolume%s-%03d.spi"%(self.timestamp, self.params['numiters']))
		emancmd = "proc3d %s %s"%(finalrawvolfile, "rmeasure.mrc")
		apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
		apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin']
		self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc")
		#apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan")
		apFile.removeFile("rmeasure.mrc")
Beispiel #50
0
def emanMrcToStack(partlist):
    apFile.removeStack("emanmrc.hed", warn=False)
    for part in partlist:
        mrc.write(part, "temp.mrc")
        emancmd = "proc2d temp.mrc emanmrc.hed"
        apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
        apFile.removeFile("temp.mrc")
    return
Beispiel #51
0
def emanSpiderToStack(partlist):
    apFile.removeStack("emanspi.hed", warn=False)
    for part in partlist:
        spider.write(part, "temp.spi")
        emancmd = "proc2d temp.spi emanspi.hed"
        apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
        apFile.removeFile("temp.spi")
    return
    def start(self):
        self.stackdata = appiondata.ApStackData.direct_query(
            self.params['stackid'])
        if self.params['vertical'] is not True:
            self.alignstackdata = appiondata.ApAlignStackData.direct_query(
                self.params['alignstackid'])

        # Path of the stack
        self.stackdata = apStack.getOnlyStackData(self.params['stackid'])
        fn_oldstack = os.path.join(self.stackdata['path']['path'],
                                   self.stackdata['name'])

        rotfile = None
        if self.params['vertical'] is not True:
            # get averaged image:
            self.alignstackdata = appiondata.ApAlignStackData.direct_query(
                self.params['alignstackid'])
            avgimg = os.path.join(self.alignstackdata['path']['path'],
                                  self.alignstackdata['avgmrcfile'])

            # Convert averaged aligned mrcfile to spider
            spiavg = os.path.join(self.params['rundir'], "avg.spi")
            emancmd = "proc2d %s %s spiderswap edgenorm" % (avgimg, spiavg)
            apEMAN.executeEmanCmd(emancmd, verbose=True)

            # find rotation for vertical alignment
            rot = self.findRotation(spiavg)
            apDisplay.printMsg("found average rotation: %.2f" % rot)

            rotlist = self.getInplaneRotations()
            rotfile = self.createRotationSpiList(rotlist, rot)

        # Convert the original stack to spider
        spistack = self.convertStackToSpider(fn_oldstack)
        # boxmask the particles
        spimaskfile = "masked" + self.timestamp + ".spi"
        self.boxMask(spistack, spimaskfile, rotfile)
        # Convert the spider stack to imagic
        imgstack = self.convertStackToImagic(spimaskfile)

        # Create average MRC
        apStack.averageStack(imgstack)

        # Clean up
        apDisplay.printMsg("deleting temporary processing files")
        os.remove(spistack)
        os.remove(spimaskfile)

        # Upload results
        if self.params['commit'] is True:
            oldstackparts = apStack.getStackParticlesFromId(
                self.params['stackid'])
            apStack.commitMaskedStack(self.params,
                                      oldstackparts,
                                      newname='start.hed')

        time.sleep(1)
        return
    def insertParticlesIntoDatabase(self, stackid, partlist):
        count = 0
        inserted = 0
        t0 = time.time()
        apDisplay.printColor("Inserting particle alignment data, please wait",
                             "cyan")
        for partdict in partlist:
            count += 1
            if count % 100 == 0:
                sys.stderr.write(".")

            ### setup reference
            refq = appiondata.ApAlignReferenceData()
            refq['refnum'] = partdict['refnum']
            refq['iteration'] = self.lastiter
            refsearch = "part" + self.params['timestamp'] + "_ref*" + str(
                partdict['refnum']) + "*"
            refbase = os.path.splitext(glob.glob(refsearch)[0])[0]
            refq['mrcfile'] = refbase + ".mrc"
            refq['path'] = appiondata.ApPathData(
                path=os.path.abspath(self.params['rundir']))
            refq['alignrun'] = self.alignstackdata['alignrun']
            if partdict['refnum'] in self.resdict:
                refq['ssnr_resolution'] = self.resdict[partdict['refnum']]
            reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
            if not os.path.isfile(reffile):
                emancmd = "proc2d " + refbase + ".xmp " + refbase + ".mrc"
                apEMAN.executeEmanCmd(emancmd, verbose=False)
            if not os.path.isfile(reffile):
                apDisplay.printError("could not find reference file: " +
                                     reffile)

            ### setup particle
            alignpartq = appiondata.ApAlignParticleData()
            alignpartq['partnum'] = partdict['partnum']
            alignpartq['alignstack'] = self.alignstackdata
            stackpartdata = apStack.getStackParticle(stackid,
                                                     partdict['partnum'])
            alignpartq['stackpart'] = stackpartdata
            alignpartq['xshift'] = partdict['xshift']
            alignpartq['yshift'] = partdict['yshift']
            alignpartq['rotation'] = partdict['inplane']
            alignpartq['mirror'] = partdict['mirror']
            alignpartq['ref'] = refq
            alignpartq['spread'] = partdict['spread']

            ### insert
            if self.params['commit'] is True:
                inserted += 1
                alignpartq.insert()

        apDisplay.printColor(
            "\ninserted " + str(inserted) + " of " + str(count) +
            " particles into the database in " +
            apDisplay.timeString(time.time() - t0), "cyan")

        return
def genProj(euler, tilt=False):
	(alt, az, phi) = euler
	rotcmd = ( "proc3d groel.mrc rotated.mrc rot="
		+str(alt)+","+str(az)+","+str(phi) )
	apEMAN.executeEmanCmd(rotcmd, verbose=False, showcmd=False)
	if tilt is True:
		cmd = "project3d rotated.mrc euler=15,90,-90"
	else:
		cmd = "project3d rotated.mrc euler=0,0,0"
	apEMAN.executeEmanCmd(cmd, verbose=False, showcmd=False)
Beispiel #55
0
	def runIMAGICclassify(self):
		bfile = "msaclassify.job"
		outfile = "classes"
		apFile.removeStack(outfile)
		numIters = int(self.params['numpart']*self.params['itermult'])
		decrement = self.params['start']-self.params['end']
		if self.params['iter']>0:
			decrement /= float(self.params['iter'])
		numClasses = self.params['start']-(decrement*self.params['currentiter'])
		stackfile = self.params['alignedstack']
		self.params['currentnumclasses'] = numClasses

		f = open(bfile,'w')
		f.write("#!/bin/csh -f\n")
		f.write("setenv IMAGIC_BATCH 1\n")
		f.write("%s/msa/classify.e <<EOF\n" % self.imagicroot)
		f.write("IMAGES\n")
		f.write("%s\n"%stackfile)
		f.write("0\n")
		f.write("%i\n"%self.params['activeeigen'])
		f.write("YES\n")
		f.write("%i\n"%numClasses)
		f.write("classes_start\n")
		f.write("EOF\n")

		f.write("%s/msa/classum.e <<EOF\n" % self.imagicroot)
		f.write("%s\n"%stackfile)
		f.write("classes_start\n")
		f.write("%s\n"%outfile)
		f.write("YES\n")
		f.write("NONE\n")
		f.write("0\n")
		if int(self.imagicversion) >= 120619:
			f.write("NONE\n") # Mode of summing statistics
		f.write("EOF\n")

		## make eigenimage stack appion-compatible
		f.write("proc2d eigenim.hed eigenim.hed inplace\n")

		f.write("touch msaclassify_done.txt\n")
		f.write("exit\n")
		f.close()

		## execute the batch file
		aligntime0 = time.time()
		apEMAN.executeEmanCmd("chmod 755 "+bfile)

		apDisplay.printColor("Running IMAGIC .batch file: %s"%(os.path.abspath(bfile)), "cyan")
		apIMAGIC.executeImagicBatchFile(os.path.abspath(bfile))

		os.remove("msaclassify_done.txt")

		if not os.path.exists(outfile+".hed"):
			apDisplay.printError("ERROR IN IMAGIC SUBROUTINE")
		apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-aligntime0), "cyan")
    def processVolume(self, spivolfile, iternum=0):
        ### set values
        apix = apStack.getStackPixelSizeFromStackId(
            self.params['tiltstackid']) * self.params['tiltbin']
        boxsize = self.getBoxSize()
        rawspifile = os.path.join(
            self.params['rundir'],
            "rawvolume%s-%03d.spi" % (self.timestamp, iternum))
        mrcvolfile = os.path.join(
            self.params['rundir'],
            "volume%s-%03d.mrc" % (self.timestamp, iternum))
        lowpass = self.params['lowpassvol']
        ### copy original to raw file
        shutil.copy(spivolfile, rawspifile)

        ### convert to mrc
        emancmd = ("proc3d " + spivolfile + " " + mrcvolfile +
                   " norm=0,1 apix=" + str(apix))
        apEMAN.executeEmanCmd(emancmd, verbose=False)

        ### median filter
        rawvol = mrc.read(mrcvolfile)
        medvol = ndimage.median_filter(rawvol, size=self.params['median'])
        mrc.write(medvol, mrcvolfile)

        ### low pass filter
        emancmd = ("proc3d " + mrcvolfile + " " + mrcvolfile +
                   " center norm=0,1 apix=" + str(apix) + " lp=" +
                   str(lowpass))
        apEMAN.executeEmanCmd(emancmd, verbose=False)

        ### set origin
        emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " origin=0,0,0 "
        apEMAN.executeEmanCmd(emancmd, verbose=False)

        ### mask volume
        emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " mask=" + str(
            self.params['radius'])
        apEMAN.executeEmanCmd(emancmd, verbose=False)

        ### convert to spider
        apFile.removeFile(spivolfile)
        emancmd = "proc3d " + mrcvolfile + " " + spivolfile + " spidersingle"
        apEMAN.executeEmanCmd(emancmd, verbose=False)

        ### image with chimera
        if self.params['skipchimera'] is False:
            if self.params['mass'] is not None:
                apDisplay.printMsg("Using scale by mass method")
                apChimera.setVolumeMass(mrcvolfile,
                                        apix=apix,
                                        mass=self.params['mass'])
            apChimera.renderSnapshots(mrcvolfile, self.params['contour'],
                                      self.params['zoom'], 'c1')
        return mrcvolfile