Esempio n. 1
0
def convertPostscriptToPng(psfile, pngfile, size=1024):

    ### better pstopnm pre-step
    pstopnmcmd = "pstopnm -xsize=2000 -ysize=2000 -xborder=0 -yborder=0 -portrait " + psfile
    proc = subprocess.Popen(pstopnmcmd, shell=True)
    proc.wait()

    ### direct conversion
    ppmfile = os.path.splitext(psfile)[0] + "001.ppm"
    if os.path.isfile(ppmfile):
        imagemagickcmd = ("convert -colorspace Gray -trim -resize " +
                          str(size) + "x" + str(size) + " " + ppmfile + " " +
                          pngfile)
    else:
        ppmfile = psfile + "001.ppm"
        if os.path.isfile(ppmfile):
            imagemagickcmd = ("convert -colorspace Gray -trim -resize " +
                              str(size) + "x" + str(size) + " " + ppmfile +
                              " " + pngfile)
        else:
            imagemagickcmd = ("convert -colorspace Gray -trim -resize " +
                              str(size) + "x" + str(size) + " " + psfile +
                              " " + pngfile)
    proc = subprocess.Popen(imagemagickcmd, shell=True)
    proc.wait()

    if os.path.isfile(ppmfile):
        apFile.removeFile(ppmfile)

    if not os.path.isfile(pngfile):
        apDisplay.printWarning("Postscript image conversion failed")
	def createTemplateStack(self):
		"""
		takes the spider file and creates an average template of all particles
		"""

		templatestack = os.path.join(self.params['rundir'], "templatestack00.spi")
		apFile.removeFile(templatestack, warn=True)

		### hack to use standard filtering library
		templateparams = {}
		templateparams['apix'] = self.stack['apix']
		templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
		templateparams['templateIds'] = self.templatelist
		templateparams['bin'] = self.params['bin']
		templateparams['lowpass'] = self.params['lowpass']
		templateparams['median'] = None
		templateparams['pixlimit'] = None
		print templateparams
		apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
		filelist = apTemplate.getTemplates(templateparams)

		for mrcfile in filelist:
			emancmd  = ("proc2d templates/"+mrcfile+" "+templatestack
				+" clip="+str(self.boxsize)+","+str(self.boxsize)
				+" spider ")
			if self.params['inverttemplates'] is True:
				emancmd += " invert "
			apEMAN.executeEmanCmd(emancmd, showcmd=False)

		return templatestack
    def xmippNormStack(self, inStackPath, outStackPath):
        ### convert stack into single spider files
        selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath)

        ### setup Xmipp command
        xmippexe = apParam.getExecPath("xmipp_normalize", die=True)
        apDisplay.printMsg("Using Xmipp to normalize particle stack")
        normtime = time.time()
        xmippopts = (" " +
                     " -i %s" % os.path.join(self.params['rundir'], selfile) +
                     " -method Ramp " + " -background circle %i" %
                     (self.stack['boxsize'] / self.params['bin'] * 0.4) +
                     " -remove_black_dust" + " -remove_white_dust" +
                     " -thr_black_dust -%.2f" % (self.params['xmipp-norm']) +
                     " -thr_white_dust %.2f" % (self.params['xmipp-norm']))
        xmippcmd = xmippexe + " " + xmippopts
        apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
        normtime = time.time() - normtime
        apDisplay.printMsg("Xmipp normalization time: " +
                           apDisplay.timeString(normtime))

        ### recombine particles to a single imagic stack
        tmpstack = "tmp.xmippStack.hed"
        apXmipp.gatherSingleFilesIntoStack(selfile, tmpstack)
        apFile.moveStack(tmpstack, outStackPath)

        ### clean up directory
        apFile.removeFile(selfile)
        apFile.removeDir("partfiles")
        def createOrientationReference(self):
                """
                convert spider orientation reference
                """

                orientref = os.path.join(self.params['rundir'], "orient.spi")
                apFile.removeFile(orientref, warn=True)

                #templatedata = apTemplate.getTemplateFromId(self.params['orientref'])
                #templatefile = os.path.join(templatedata['path']['path'], templatedata['templatename'])

                ### hack to use standard filtering library
                templateparams = {}
                templateparams['apix'] = self.stack['apix']
                templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
                templateparams['templateIds'] = [self.params['orientref'],]
                templateparams['bin'] = self.params['bin']
                templateparams['lowpass'] = self.params['lowpass']
                templateparams['median'] = None
                templateparams['pixlimit'] = None
                print 'Converting orientation reference:\n', templateparams
                apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
                filelist = apTemplate.getTemplates(templateparams)
                mrcfile = filelist[0]

                localclip = self.clipsize/self.params['bin']
                emancmd  = ("proc2d templates/"+mrcfile+" "+orientref
                        +" clip="+str(localclip)+","+str(localclip)
                        +" edgenorm spiderswap-single ")
                if self.params['inverttemplates'] is True:
                        emancmd += " invert "
                apEMAN.executeEmanCmd(emancmd, showcmd=False)

                return orientref
	def start(self):
		self.runtime = 0
		self.partlist = []
		self.stack = {}
		self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
		self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
		self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
		self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
		self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])

		self.checkNoRefRun()

		### convert stack to spider
		spiderstack = self.createSpiderFile()

		### create initialization template
		if self.params['initmethod'] == 'allaverage':
			templatefile = self.averageTemplate()
		elif self.params['initmethod'] == 'selectrand':
			templatefile = self.selectRandomParticles()
		elif self.params['initmethod'] == 'randpart':
			templatefile = self.pickRandomParticle()
		elif self.params['initmethod'] == 'template':
			templatefile = self.getTemplate()
		else:
			apDisplay.printError("unknown initialization method defined: "
				+str(self.params['initmethod'])+" not in "+str(self.initmethods))

		apDisplay.printColor("Running spider this can take awhile","cyan")

		### run the alignment
		aligntime = time.time()
		pixrad = int(round(self.params['partrad']/self.stack['apix']/self.params['bin']))
		alignedstack, self.partlist = alignment.refFreeAlignParticles(
			spiderstack, templatefile,
			self.params['numpart'], pixrad,
			self.params['firstring'], self.params['lastring'],
			rundir = ".")
		aligntime = time.time() - aligntime
		apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))

		### remove large, worthless stack
		spiderstack = os.path.join(self.params['rundir'], "start.spi")
		apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
		apFile.removeFile(spiderstack, warn=False)

		### convert stack to imagic
		imagicstack = self.convertSpiderStack(alignedstack)
		apFile.removeFile(alignedstack)

		inserttime = time.time()
		if self.params['commit'] is True:
			self.runtime = aligntime
			self.insertNoRefRun(imagicstack, insert=True)
		else:
			apDisplay.printWarning("not committing results to DB")
		inserttime = time.time() - inserttime

		apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
		apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
	def checkAndCopyFiles(self):
		if not os.path.isfile(self.params['inputstack']):
			apDisplay.printError("Stack file %s does not exist"%(self.params['inputstack']))
		if not os.path.isfile(self.params['inputvol']):
			apDisplay.printError("Initial model file %s does not exist"%(self.params['inputvol']))
		if not os.path.isfile(self.params['inputparam']):
			apDisplay.printError("Initial parameter file %s does not exist"%(self.params['inputparam']))

		apParam.createDirectory(self.params['rundir'])

		firstVolFile = os.path.join(self.params['rundir'], "initial_volume.mrc")
		apDisplay.printColor("Linking initial model %s to recon folder %s"%
			(self.params['inputvol'], firstVolFile), "purple")
		if not os.path.exists(firstVolFile):
			os.symlink(self.params['inputvol'], firstVolFile)
			#shutil.copy(self.params['inputvol'], firstVolFile)

		firstParamFile = os.path.join(self.params['rundir'], "initial_params.par")		
		apDisplay.printColor("Copying initial parameters %s to recon folder %s"
			%(self.params['inputparam'], firstParamFile), "purple")
		apFile.removeFile(firstParamFile)
		time.sleep(0.1)			
		shutil.copy(self.params['inputparam'], firstParamFile)

		os.chdir(self.params['rundir'])

		for side in ('left', 'right'):
			prefix = "%s.iter%02d"%(side, 0)
			volFile = "threed.%s.mrc"%(prefix)
			if not os.path.islink(volFile):
				os.symlink(os.path.basename(firstVolFile), volFile)

		self.createSplitParamFiles(firstParamFile)

		return
        def createTemplateStack(self):
                """
                convert spider template stack
                """

                templatestack = os.path.join(self.params['rundir'], "templatestack.spi")
                apFile.removeFile(templatestack, warn=True)

                ### hack to use standard filtering library
                templateparams = {}
                templateparams['apix'] = self.stack['apix']
                templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
                templateparams['templateIds'] = self.templatelist
                templateparams['bin'] = self.params['bin']
                templateparams['lowpass'] = self.params['lowpass']
                templateparams['median'] = None
                templateparams['pixlimit'] = None
                print 'Converting reference templates:\n', templateparams
                apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
                filelist = apTemplate.getTemplates(templateparams)

                localclip = self.clipsize/self.params['bin']
                for mrcfile in filelist:
                        emancmd  = ("proc2d templates/"+mrcfile+" "+templatestack
                                +" clip="+str(localclip)+","+str(localclip)
                                +" edgenorm spiderswap ")
                        if self.params['inverttemplates'] is True:
                                emancmd += " invert "
                        apEMAN.executeEmanCmd(emancmd, showcmd=False)

                return templatestack
def stackToSpiderStack(stack,spiderstack,apix,boxsize,lp=0,hp=0,bin=1,numpart=0):
	"""
	convert an input stack (i.e. imagic) and write it out to a spider-formatted stack
	"""
	emancmd  = "proc2d "
	if not os.path.isfile(stack):
		apDisplay.printError("stackfile does not exist: "+stack)
	emancmd += stack+" "

	apFile.removeFile(spiderstack, warn=True)
	emancmd += spiderstack+" "

	emancmd += "apix="+str(apix)+" "
	if lp > 0:
		emancmd += "lp="+str(lp)+" "
	if hp > 0:
		emancmd += "hp="+str(hp)+" "
	if bin > 1:
		clipboxsize = boxsize*bin
		emancmd += "shrink="+str(bin)+" "
		emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" "
	if numpart > 0:
		emancmd += "last="+str(numpart-1)+" "
	emancmd += "spiderswap edgenorm"
	starttime = time.time()
	apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
	from appionlib import apEMAN
	apEMAN.executeEmanCmd(emancmd, verbose=True)
	apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return
def convertPostscriptToPng(psfile, pngfile, size=1024):

        ### better pstopnm pre-step
        pstopnmcmd = "pstopnm -xsize=2000 -ysize=2000 -xborder=0 -yborder=0 -portrait "+psfile
        proc = subprocess.Popen(pstopnmcmd, shell=True)
        proc.wait()

        ### direct conversion
        ppmfile = os.path.splitext(psfile)[0]+"001.ppm"
        if os.path.isfile(ppmfile):
                imagemagickcmd = ("convert -colorspace Gray -trim -resize "
                        +str(size)+"x"+str(size)+" "+ppmfile+" "+pngfile)
        else:
                ppmfile = psfile+"001.ppm"
                if os.path.isfile(ppmfile):
                        imagemagickcmd = ("convert -colorspace Gray -trim -resize "
                                +str(size)+"x"+str(size)+" "+ppmfile+" "+pngfile)
                else:
                        imagemagickcmd = ("convert -colorspace Gray -trim -resize "
                                +str(size)+"x"+str(size)+" "+psfile+" "+pngfile)
        proc = subprocess.Popen(imagemagickcmd, shell=True)
        proc.wait()

        if os.path.isfile(ppmfile):
                apFile.removeFile(ppmfile)

        if not os.path.isfile(pngfile):
                apDisplay.printWarning("Postscript image conversion failed")
Esempio n. 10
0
    def processTemplate(self, mrcfile):
        ### shrink
        apDisplay.printMsg("Binning template by a factor of " +
                           str(self.params['bin']))
        clipsize = int(
            math.floor(self.stack['boxsize'] / self.params['bin']) *
            self.params['bin'])
        emancmd = ("proc2d " + mrcfile + " " + mrcfile + " shrink=" +
                   str(self.params['bin']) + " spiderswap ")
        emancmd += "clip=" + str(clipsize) + "," + str(clipsize) + " "
        apEMAN.executeEmanCmd(emancmd)

        ### normalize and center
        #apDisplay.printMsg("Normalize and centering template")
        #emancmd = "proc2d "+mrcfile+" "+mrcfile+" edgenorm center"
        #apEMAN.executeEmanCmd(emancmd)

        ### convert to SPIDER
        apDisplay.printMsg("Converting template to SPIDER")
        templatefile = "template.spi"
        if os.path.isfile(templatefile):
            apFile.removeFile(templatefile, warn=True)
        emancmd = "proc2d template.mrc " + templatefile + " spiderswap "
        apEMAN.executeEmanCmd(emancmd)

        return templatefile
Esempio n. 11
0
    def convertSpiderStack(self, spiderstack):
        """
		takes the stack file and creates a spider file ready for processing
		"""
        emancmd = "proc2d "
        if not os.path.isfile(spiderstack):
            apDisplay.printError("stackfile does not exist: " + spiderstack)
        emancmd += spiderstack + " "

        imagicstack = os.path.join(self.params['rundir'], "alignstack.hed")
        apFile.removeFile(imagicstack, warn=True)
        emancmd += imagicstack + " "

        starttime = time.time()
        apDisplay.printColor(
            "Running spider stack conversion this can take a while", "cyan")
        apEMAN.executeEmanCmd(emancmd, verbose=True)
        apDisplay.printColor(
            "finished eman in " +
            apDisplay.timeString(time.time() - starttime), "cyan")

        numpart = apFile.numImagesInStack(imagicstack)
        if numpart != self.params['numpart']:
            apDisplay.printError(
                "The number of particles returned is different from the number requested"
            )

        return imagicstack
        def makeEvenOddClasses(self, listfile, outputstack, classdata, maskrad):
                f=open(listfile,'r')
                f.readline()
                lines = f.readlines()
                f.close()
                randstr = str(int(random.random()*10e5))
                evenfile = self.rootname+"-even.lst"
                evenf = open(evenfile,'w')
                oddfile = self.rootname+"-odd.lst"
                oddf = open(oddfile,'w')
                evenf.write("#LST\n")
                oddf.write("#LST\n")
                neven=0
                nodd=0
                for i in range(0, len(lines)):
                        if i%2:
                                nodd+=1
                                oddf.write(lines[i])
                        else:
                                neven+=1
                                evenf.write(lines[i])
                evenf.close()
                oddf.close()

                if neven>0:
                        self.makeClassAverages(evenfile, self.params['evenstack'], classdata, maskrad)
                if nodd>0:
                        self.makeClassAverages(oddfile, self.params['oddstack'], classdata, maskrad)
                apFile.removeFile(evenfile)
                apFile.removeFile(oddfile)
Esempio n. 13
0
    def prepareStack(self):
        ### setup box and mask sizes
        self.alignstackdata = self.getAlignedStack()
        maskpixrad = self.params['maskrad'] / self.alignstackdata[
            'pixelsize'] / self.params['bin']
        boxpixdiam = int(math.ceil(maskpixrad) + 1) * 2
        if boxpixdiam * self.params['bin'] > self.alignstackdata['boxsize']:
            boxpixdiam = math.floor(self.alignstackdata['boxsize'] /
                                    self.params['bin'])
        clippixdiam = boxpixdiam * self.params['bin']
        apDisplay.printMsg("Pixel mask radius=" + str(maskpixrad))

        ### convert aligned stack to local dir
        oldalignedstack = os.path.join(self.alignstackdata['path']['path'],
                                       self.alignstackdata['imagicfile'])
        alignedstackname = "alignedstack.hed"
        alignedstack = os.path.join(self.params['rundir'], alignedstackname)
        apFile.removeFile(alignedstack)
        emancmd = ("proc2d %s %s shrink=%d clip=%d,%d" %
                   (oldalignedstack, alignedstack, self.params['bin'],
                    clippixdiam, clippixdiam))
        if self.params['numpart'] is not None and self.params[
                'numpart'] < self.numpart:
            emancmd += " last=%d" % (self.params['numpart'] - 1)
            self.numpart = self.params['numpart']
        apEMAN.executeEmanCmd(emancmd, verbose=True)
        return alignedstack
Esempio n. 14
0
def backProjection(stack,
                   select,
                   ang,
                   out,
                   sym=None,
                   nproc=1,
                   dataext=".spi",
                   inMySpi=False):
    if inMySpi is False:
        mySpi = spyder.SpiderSession(nproc=nproc,
                                     dataext=dataext,
                                     logo=False,
                                     log=False)
    else:
        mySpi = inMySpi
    apFile.removeFile(out)

    if sym is None:
        sym = "*"
    mySpi.toSpiderQuiet("BP 3F",
                        spyder.fileFilter(stack) + "@******",
                        spyder.fileFilter(select), spyder.fileFilter(ang),
                        spyder.fileFilter(sym), spyder.fileFilter(out))
    if inMySpi is False:
        mySpi.close()
 def uploadImages(self):
     ### get simulated instrument ids
     scopeid, cameraid = self.getInstrumentIds()
     ### Download images
     imglist = self.downloadImagesFromAMI()
     ### create batch file
     imagedatfile = self.createImageBatchFile(imglist)
     ### run command
     script = os.path.join(self.appiondir, "bin", "imageloader.py ")
     params = (
         " --runname=%s --projectid=%d --session=%s --batch=%s --scopeid=%d --cameraid=%d --description='%s' "
         % (
             self.sessionname,
             self.params["projectid"],
             self.sessionname,
             imagedatfile,
             scopeid,
             cameraid,
             self.params["description"],
         )
     )
     if self.params["commit"] is True:
         params += " --commit "
     else:
         params += " --no-commit "
     self.runCommand(script + " " + params)
     for imgname in imglist:
         apFile.removeFile(imgname)
        def start(self):
                #get aligned stack id
                aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
                xSizeVoxel = aligndata['boxsize']
                #get averaged image
                avgmrc = os.path.join(aligndata['path']['path'], aligndata["avgmrcfile"])
                avg = mrc.read(avgmrc)
                tmpSpiderFile="average.xmp"
                spider.write(avg, tmpSpiderFile)
                self.runFindCenter(tmpSpiderFile,xSizeVoxel)
                #get aligned stack
                alignStack = os.path.join(aligndata['path']['path'], aligndata["imagicfile"])
                tempFileNameforSpectra=self.runMakeSpectra(alignStack)

                #kerdensom will work with spectra output
                self.runKerdenSOM(tempFileNameforSpectra)
                self.createMontageInMemory()
                self.insertRotKerDenSOM()

                #apFile.removeFile(outdata)
                apFile.removeFilePattern("*.cod")
                apFile.removeFilePattern("*.err")
                apFile.removeFilePattern("*.his")
                apFile.removeFilePattern("*.inf")
                apFile.removeFilePattern("*.vs")
                apFile.removeFilePattern("*.xmp")
                apFile.removeFile(tempFileNameforSpectra)
Esempio n. 17
0
def peakTreeToPikFile(peaktree, imgname, tmpl, rundir="."):
    outpath = os.path.join(rundir, "pikfiles")
    apParam.createDirectory(outpath, warning=False)
    outfile = os.path.join(outpath, imgname + "." + str(tmpl) + ".pik")
    apFile.removeFile(outfile, warn=True)
    #WRITE PIK FILE
    f = open(outfile, 'w')
    f.write(
        "#filename x y mean stdev corr_coeff peak_size templ_num angle moment diam\n"
    )
    for peakdict in peaktree:
        row = peakdict['ycoord']
        col = peakdict['xcoord']
        if 'corrcoeff' in peakdict:
            rho = peakdict['corrcoeff']
        else:
            rho = 1.0
        size = peakdict['peakarea']
        mean_str = "%.4f" % peakdict['correlation']
        std_str = "%.4f" % peakdict['peakstddev']
        mom_str = "%.4f" % peakdict['peakmoment']
        if peakdict['diameter'] is not None:
            diam_str = "%.2f" % peakdict['diameter']
        else:
            diam_str = "0"
        if 'template' in peakdict:
            tmplnum = peakdict['template']
        else:
            tmplnum = tmpl
        #filename x y mean stdev corr_coeff peak_size templ_num angle moment
        out = imgname+".mrc "+str(int(col))+" "+str(int(row))+ \
         " "+mean_str+" "+std_str+" "+str(rho)+" "+str(int(size))+ \
         " "+str(tmplnum)+" 0 "+mom_str+" "+diam_str
        f.write(str(out) + "\n")
    f.close()
def ClCla(alignedstack, numpart=None, numclasses=40,
		factorlist=range(1,5), corandata="coran/corandata", dataext=".spi"):
	"""
	this doesn't work
	"""
	if alignedstack[-4:] == dataext:
		alignedstack = alignedstack[:-4]

	rundir = "cluster"
	classavg = rundir+"/"+("classavgstack%03d" % numclasses)
	classvar = rundir+"/"+("classvarstack%03d" % numclasses)
	apParam.createDirectory(rundir)
	for i in range(numclasses):
		apFile.removeFile(rundir+("/classdoc%04d" % (i+1))+dataext)
	apFile.removeFile(rundir+"/clusterdoc"+dataext)

	factorstr, factorkey = operations.intListToString(factorlist)

	### do hierarchical clustering
	mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
	mySpider.toSpider(
		"CL CLA",
		corandata, # path to coran data
		rundir+"/clusterdoc",	#clusterdoc file
		factorstr, #factor numbers
		"5,8",
		"4",
		"2", # minimum number of particles per class
		"Y", rundir+"/dendrogram.ps",
		"Y", rundir+"/dendrogramdoc",
	)
	mySpider.close()
Esempio n. 19
0
def symmetryDoc(symtype, symfold=None, outfile="sym.spi", dataext=".spi"):
    mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
    apFile.removeFile(outfile)
    mySpi.toSpiderQuiet("SY", spyder.fileFilter(outfile), symtype)
    if symfold is not None:
        mySpi.toSpiderQuiet(symfold)
    mySpi.close()
def backprojectCG(stackfile, eulerdocfile, volfile, numpart, pixrad, dataext=".spi"):
	"""
	inputs:
		stack, in spider format
		eulerdocfile
	outputs:
		volume
	"""
	### setup
	starttime = time.time()
	stackfile = spyder.fileFilter(stackfile)
	eulerdocfile = spyder.fileFilter(eulerdocfile)
	volfile = spyder.fileFilter(volfile)
	if not os.path.isfile(stackfile+dataext):
		apDisplay.printError("stack file not found: "+stackfile+dataext)
	if not os.path.isfile(eulerdocfile+dataext):
		apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
	apFile.removeFile(volfile+dataext)
	nproc = apParam.getNumProcessors()
	mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
	mySpider.toSpider("BP CG", 
		stackfile+"@*****", #stack file
		"1-%d"%(numpart), #number of particles
		str(pixrad), #particle radius
		eulerdocfile, #angle doc file
		"N", #has symmetry?, does not work
		volfile, #filename for volume
 		"%.1e,%.1f" % (1.0e-5, 0.0), #error, chi^2 limits
 		"%d,%d" % (25,1), #iterations, 1st derivative mode
 		"2000", #lambda - higher=less sensitive to noise
	)
	mySpider.close()
	apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return
Esempio n. 21
0
    def processParticles(self, imgdata, partdatas, shiftdata):
        # need to avoid non-frame saved image for proper caching
        if imgdata is None or imgdata['camera']['save frames'] != True:
            apDisplay.printWarning('%s skipped for no-frame-saved\n ' %
                                   imgdata['filename'])
            return

        ### first remove any existing files
        rundir = self.params['rundir']
        imgname = imgdata['filename']
        moviegrouppath = os.path.join(rundir,
                                      '%s_*.%s' % (imgname, self.movieformat))
        apFile.removeFile(moviegrouppath)

        shortname = apDisplay.short(imgdata['filename'])

        apFile.removeFile(os.path.join(rundir, '%s*.jpg' % (shortname)))

        ### set processing image
        try:
            self.dd.setImageData(imgdata)
        except Exception, e:
            raise
            apDisplay.printWarning('%s: %s' % (e.__class__.__name__, e))
            return
Esempio n. 22
0
def assemblePngs(keys, tag, montagestack):
	apDisplay.printMsg("assembling pngs into montage")
	# get path from montagestack
	stackpath = os.path.dirname(os.path.abspath(montagestack))

	montagecmd = "montage -geometry +4+4 "
	montagestackdata = apImagicFile.readImagic(montagestack)
	for i,key in enumerate(keys):
		if i % 20 == 0:
			sys.stderr.write(".")
		pngfile = key+".png"
		array = montagestackdata['images'][i]
		apImage.arrayToPng(array, pngfile, normalize=True, msg=False)
		#proccmd = "proc2d "+montagestack+" "+pngfile+" first="+str(i)+" last="+str(i)
		#apEMAN.executeEmanCmd(proccmd, verbose=False, showcmd=False)
		montagecmd += pngfile+" "
	apDisplay.printMsg("montaging")
	montagefile = os.path.join(stackpath,"montage"+tag+".png")
	montagecmd += montagefile
	apEMAN.executeEmanCmd(montagecmd, verbose=True)
	#rotatecmd = "mogrify -rotate 180 -flop "+montagefile
	#apEMAN.executeEmanCmd(rotatecmd, verbose=False)
	for key in keys:
		apFile.removeFile(key+".png")
	if not os.path.isfile(montagefile):
		apDisplay.printWarning("failed to create montage file")
def projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext=".spi"):
	"""
	project 3D volumes using given Euler angles
	"""
	starttime = time.time()

	volfile = spyder.fileFilter(volfile)
	eulerdocfile = spyder.fileFilter(eulerdocfile)
	projstackfile = spyder.fileFilter(projstackfile)
	if not os.path.isfile(volfile+dataext):
		apDisplay.printError("volume file not found: "+volfile+dataext)
	if not os.path.isfile(eulerdocfile+dataext):
		apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)

	apFile.removeFile(projstackfile)
	nproc = apParam.getNumProcessors()
	mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
	mySpider.toSpider("PJ 3Q", 
		volfile, #input vol file
		str(pixrad), #pixel radius
		"1-%d"%(numpart), #number of particles		
		eulerdocfile, #Euler DOC file
		projstackfile+"@*****", #output projections
	)
	mySpider.close()
	apDisplay.printColor("finished projections in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return
def peakTreeToPikFile(peaktree, imgname, tmpl, rundir="."):
	outpath = os.path.join(rundir, "pikfiles")
	apParam.createDirectory(outpath, warning=False)
	outfile = os.path.join(outpath, imgname+"."+str(tmpl)+".pik")
	apFile.removeFile(outfile, warn=True)
	#WRITE PIK FILE
	f=open(outfile, 'w')
	f.write("#filename x y mean stdev corr_coeff peak_size templ_num angle moment diam\n")
	for peakdict in peaktree:
		row = peakdict['ycoord']
		col = peakdict['xcoord']
		if 'corrcoeff' in peakdict:
			rho = peakdict['corrcoeff']
		else:
			rho = 1.0
		size = peakdict['peakarea']
		mean_str = "%.4f" % peakdict['correlation']
		std_str = "%.4f" % peakdict['peakstddev']
		mom_str = "%.4f" % peakdict['peakmoment']
		if peakdict['diameter'] is not None:
			diam_str = "%.2f" % peakdict['diameter']
		else:
			diam_str = "0"
		if 'template' in peakdict:
			tmplnum = peakdict['template']
		else:
			tmplnum = tmpl
		#filename x y mean stdev corr_coeff peak_size templ_num angle moment
		out = imgname+".mrc "+str(int(col))+" "+str(int(row))+ \
			" "+mean_str+" "+std_str+" "+str(rho)+" "+str(int(size))+ \
			" "+str(tmplnum)+" 0 "+mom_str+" "+diam_str
		f.write(str(out)+"\n")
	f.close()
	def processImage(self, imgdata):
		self.ctfvalues = None
		fftpath = os.path.join(self.powerspecdir, apDisplay.short(imgdata['filename'])+'.powerspec.mrc')
		self.processAndSaveFFT(imgdata, fftpath)
		self.runRefineCTF(imgdata, fftpath)
		apFile.removeFile(fftpath)
		return
	def createSpiderFile(self):
		"""
		takes the stack file and creates a spider file ready for processing
		"""
		emancmd  = "proc2d "
		if not os.path.isfile(self.stack['file']):
			apDisplay.printError("stackfile does not exist: "+self.stack['file'])
		emancmd += self.stack['file']+" "

		spiderstack = os.path.join(self.params['rundir'], "start.spi")
		apFile.removeFile(spiderstack, warn=True)
		emancmd += spiderstack+" "

		emancmd += "apix="+str(self.stack['apix'])+" "
		if self.params['lowpass'] > 0:
			emancmd += "lp="+str(self.params['lowpass'])+" "
		if self.params['highpass'] > 0:
			emancmd += "hp="+str(self.params['highpass'])+" "
		if self.params['bin'] > 1:
			clipboxsize = self.boxsize*self.params['bin']
			emancmd += "shrink="+str(self.params['bin'])+" "
			emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" "
		emancmd += "last="+str(self.params['numpart']-1)+" "
		emancmd += "spider edgenorm"
		starttime = time.time()
		apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
		apEMAN.executeEmanCmd(emancmd, verbose=True)
		apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
		return spiderstack
        def convertPDBtoMRC(self, pdbfile):
                ### create density from pdb
                spidername = self.params['name']+".spi"
                mrcname = self.params['name']+".mrc"

                if self.params['method'] == 'spider':
                        if self.params['bunit'] is True:
                                pdbfile = self.removeModelFlags(pdbfile)
                        volFun.pdb2vol(pdbfile, self.params['apix'], self.params['boxsize'], spidername)
                        if not os.path.isfile(spidername):
                                apDisplay.printError("SPIDER did not create density file: "+spidername)
                        ### convert spider to mrc format
                        apDisplay.printMsg("converting spider file to mrc")
                        emancmd='proc3d %s %s apix=%.4f lp=%.2f norm=0,1 origin=0,0,0' % (spidername, mrcname,
                                self.params['apix'], self.params['lowpass'])
                        apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
                        apFile.removeFile(spidername)

                elif self.params['method'] == 'eman':
                        mult = 1.0
                        ### use resolution of mult*apix, because this is slow and lowpass is fast
                        emancmd='pdb2mrc %s %s apix=%.4f res=%.2f box=%d center allmdl' % (pdbfile, mrcname,
                                self.params['apix'], mult*self.params['apix'], self.params['boxsize'])
                        apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
                        ### lowpass to desired resolution minus the res factor specified
                        lp = math.sqrt(self.params['lowpass']**2 - (mult*self.params['apix'])**2)
                        emancmd='proc3d %s %s apix=%.4f lp=%.2f norm=0,1 origin=0,0,0' % (mrcname, mrcname,
                                self.params['apix'], lp)
                        apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)

                if self.params['viper2eman'] is True:
                        apVolume.viper2eman(mrcname, mrcname, apix=self.params['apix'])

                return mrcname
Esempio n. 28
0
def symmetryDoc(symtype,symfold=None,outfile="sym.spi",dataext=".spi"):
	mySpi=spyder.SpiderSession(dataext=dataext,logo=False,log=False)
	apFile.removeFile(outfile)
	mySpi.toSpiderQuiet("SY",spyder.fileFilter(outfile),symtype)
	if symfold is not None:
		mySpi.toSpiderQuiet(symfold)
	mySpi.close()
Esempio n. 29
0
	def convertPDBtoMRC(self, pdbfile):
		### create density from pdb
		spidername = self.params['name']+".spi"
		mrcname = self.params['name']+".mrc"

		if self.params['method'] == 'spider':
			if self.params['bunit'] is True:
				pdbfile = self.removeModelFlags(pdbfile)
			volFun.pdb2vol(pdbfile, self.params['apix'], self.params['boxsize'], spidername)
			if not os.path.isfile(spidername):
				apDisplay.printError("SPIDER did not create density file: "+spidername)
			### convert spider to mrc format
			apDisplay.printMsg("converting spider file to mrc")
			emancmd='proc3d %s %s apix=%.4f lp=%.2f norm=0,1 origin=0,0,0' % (spidername, mrcname,
				self.params['apix'], self.params['lowpass'])
			apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
			apFile.removeFile(spidername)

		elif self.params['method'] == 'eman':
			mult = 1.0
			### use resolution of mult*apix, because this is slow and lowpass is fast
			emancmd='pdb2mrc %s %s apix=%.4f res=%.2f box=%d center allmdl' % (pdbfile, mrcname,
				self.params['apix'], mult*self.params['apix'], self.params['boxsize'])
			apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
			### lowpass to desired resolution minus the res factor specified
			lp = math.sqrt(self.params['lowpass']**2 - (mult*self.params['apix'])**2)
			emancmd='proc3d %s %s apix=%.4f lp=%.2f norm=0,1 origin=0,0,0' % (mrcname, mrcname,
				self.params['apix'], lp)
			apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)

		if self.params['viper2eman'] is True:
			apVolume.viper2eman(mrcname, mrcname, apix=self.params['apix'])

		return mrcname
        def xmippNormStack(self, inStackPath, outStackPath):
                ### convert stack into single spider files
                selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath)      

                ### setup Xmipp command
                xmippexe = apParam.getExecPath("xmipp_normalize", die=True)
                apDisplay.printMsg("Using Xmipp to normalize particle stack")
                normtime = time.time()
                xmippopts = ( " "
                        +" -i %s"%os.path.join(self.params['rundir'],selfile)
                        +" -method Ramp "
                        +" -background circle %i"%(self.stack['boxsize']/self.params['bin']*0.4)
                        +" -remove_black_dust"
                        +" -remove_white_dust"
                        +" -thr_black_dust -%.2f"%(self.params['xmipp-norm'])
                        +" -thr_white_dust %.2f"%(self.params['xmipp-norm'])
                )
                xmippcmd = xmippexe+" "+xmippopts
                apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
                normtime = time.time() - normtime
                apDisplay.printMsg("Xmipp normalization time: "+apDisplay.timeString(normtime))

                ### recombine particles to a single imagic stack
                tmpstack = "tmp.xmippStack.hed"
                apXmipp.gatherSingleFilesIntoStack(selfile,tmpstack)
                apFile.moveStack(tmpstack,outStackPath)

                ### clean up directory
                apFile.removeFile(selfile)
                apFile.removeDir("partfiles")
def backproject3F(stackfile, eulerdocfile, volfile, numpart, dataext=".spi"):
	"""
	inputs:
		stack, in spider format
		eulerdocfile
	outputs:
		volume
	"""
	### setup
	starttime = time.time()
	stackfile = spyder.fileFilter(stackfile)
	eulerdocfile = spyder.fileFilter(eulerdocfile)
	volfile = spyder.fileFilter(volfile)
	if not os.path.isfile(stackfile+dataext):
		apDisplay.printError("stack file not found: "+stackfile+dataext)
	if not os.path.isfile(eulerdocfile+dataext):
		apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
	apFile.removeFile(volfile+dataext)
	nproc = apParam.getNumProcessors()
	mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
	mySpider.toSpider("BP 3F", 
		stackfile+"@*****", #stack file
		"1-%d"%(numpart), #number of particles
		eulerdocfile, #angle doc file
		"*", #input symmetry file, '*' for skip
		volfile, #filename for volume
	)
	mySpider.close()
	apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return
Esempio n. 32
0
    def start(self):
        #get aligned stack id
        aligndata = appiondata.ApAlignStackData.direct_query(
            self.params['alignstackid'])
        xSizeVoxel = aligndata['boxsize']
        #get averaged image
        avgmrc = os.path.join(aligndata['path']['path'],
                              aligndata["avgmrcfile"])
        avg = mrc.read(avgmrc)
        tmpSpiderFile = "average.xmp"
        spider.write(avg, tmpSpiderFile)
        self.runFindCenter(tmpSpiderFile, xSizeVoxel)
        #get aligned stack
        alignStack = os.path.join(aligndata['path']['path'],
                                  aligndata["imagicfile"])
        tempFileNameforSpectra = self.runMakeSpectra(alignStack)

        #kerdensom will work with spectra output
        self.runKerdenSOM(tempFileNameforSpectra)
        self.createMontageInMemory()
        self.insertRotKerDenSOM()

        #apFile.removeFile(outdata)
        apFile.removeFilePattern("*.cod")
        apFile.removeFilePattern("*.err")
        apFile.removeFilePattern("*.his")
        apFile.removeFilePattern("*.inf")
        apFile.removeFilePattern("*.vs")
        apFile.removeFilePattern("*.xmp")
        apFile.removeFile(tempFileNameforSpectra)
Esempio n. 33
0
def backProjection(
		stack,
		select,
		ang,
		out,
		sym=None,
		nproc=1,
		dataext=".spi",
		inMySpi=False):
	if inMySpi is False:
		mySpi = spyder.SpiderSession(nproc=nproc,dataext=dataext,logo=False,log=False)
	else:
		mySpi=inMySpi
	apFile.removeFile(out)

	if sym is None:
		sym="*"
	mySpi.toSpiderQuiet(
		"BP 3F",
		spyder.fileFilter(stack)+"@******",
		spyder.fileFilter(select),
		spyder.fileFilter(ang),
		spyder.fileFilter(sym),
		spyder.fileFilter(out)
	)
	if inMySpi is False:
		mySpi.close()
    def createSpiderFile(self):
        """
                takes the stack file and creates a spider file ready for processing
                """
        emancmd = "proc2d "
        if not os.path.isfile(self.stack["file"]):
            apDisplay.printError("stackfile does not exist: " + self.stack["file"])
        emancmd += self.stack["file"] + " "

        spiderstack = os.path.join(self.params["rundir"], "start.spi")
        apFile.removeFile(spiderstack, warn=True)
        emancmd += spiderstack + " "

        emancmd += "apix=" + str(self.stack["apix"]) + " "
        if self.params["lowpass"] > 0:
            emancmd += "lp=" + str(self.params["lowpass"]) + " "
        emancmd += "last=" + str(self.params["numpart"] - 1) + " "
        emancmd += "shrink=" + str(self.params["bin"]) + " "
        clipsize = int(math.floor(self.stack["boxsize"] / self.params["bin"] / 2.0) * self.params["bin"] * 2)
        emancmd += "clip=" + str(clipsize) + "," + str(clipsize) + " "
        emancmd += "spiderswap edgenorm"
        starttime = time.time()
        apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
        apEMAN.executeEmanCmd(emancmd, verbose=True)
        apDisplay.printColor("finished eman in " + apDisplay.timeString(time.time() - starttime), "cyan")
        return spiderstack
Esempio n. 35
0
	def start(self):
		aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
		boxsize = aligndata['boxsize']
		apix = aligndata['pixelsize']
		maskpixrad = self.params['maskrad']/apix
		if maskpixrad*2 > boxsize-2:
			apDisplay.printError("Mask radius is too big for boxsize: %d > %d"%(maskpixrad*2,boxsize-2))
		apDisplay.printMsg("Mask radius and boxsize: %.1f < %d"%(maskpixrad*2,boxsize-2))
		self.instack = os.path.join(aligndata['path']['path'], aligndata['imagicfile'])
		outdata = "stack.data"

		apXmipp.convertStackToXmippData(self.instack, outdata, maskpixrad,
			boxsize, numpart=self.params['numpart']-1)

		self.runKerdenSOM(outdata)
		if apFile.stackSize(self.instack) > 3.0*(1024**3):
			# Big stacks use eman
			self.createMontageByEMAN()
			binned = None
		else:
			binned = self.createMontageInMemory(apix)
		self.insertKerDenSOM(binned=binned)

		apFile.removeFile(outdata)
		apFile.removeFilePattern("*.cod")
def rctParticleShift(volfile, origstackfile, eulerdocfile, iternum, numpart, pixrad, timestamp, dataext=".spi"):
	"""
	inputs:
		stack, in spider format
		eulerdocfile
	outputs:
		volume
	"""
	starttime = time.time()
	### create corresponding projections
	projstackfile = "projstack%s-%03d.spi"%(timestamp, iternum)
	projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext)

	### clean up files
	ccdocfile = "ccdocfile%s-%03d.spi"%(timestamp, iternum)
	apFile.removeFile(ccdocfile)
	alignstackfile = "alignstack%s-%03d.spi"%(timestamp, iternum)
	apFile.removeFile(alignstackfile)

	### align particles to projection
	apDisplay.printMsg("Shifting particles")
	crossCorrelateAndShift(origstackfile, projstackfile, alignstackfile, ccdocfile, numpart)

	if not os.path.isfile(alignstackfile):
		apDisplay.printError("aligned stack file not found: "+alignstackfile)
	apDisplay.printColor("finished correlations in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return alignstackfile
Esempio n. 37
0
	def convertStackToSpider(self, emanstackfile, classnum):
		"""
		takes the stack file and creates a spider file ready for processing
		"""
		if not os.path.isfile(emanstackfile):
			apDisplay.printError("stackfile does not exist: "+emanstackfile)

		### first high pass filter particles
		apDisplay.printMsg("pre-filtering particles")
		apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
		emancmd = ("proc2d "+emanstackfile+" "+emanstackfile
			+" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
			+" inplace")
		apEMAN.executeEmanCmd(emancmd, verbose=True)

		### convert imagic stack to spider
		emancmd  = "proc2d "
		emancmd += emanstackfile+" "
		spiderstack = os.path.join(self.params['rundir'], str(classnum), "otrstack"+self.timestamp+".spi")
		apFile.removeFile(spiderstack, warn=True)
		emancmd += spiderstack+" "

		emancmd += "spiderswap edgenorm"
		starttime = time.time()
		apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
		apEMAN.executeEmanCmd(emancmd, verbose=True)
		apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
		return spiderstack
Esempio n. 38
0
	def runRmeasure(self, volfile):
		emancmd = "proc3d %s %s"%(volfile, "rmeasure.mrc")
		apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
		apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin']
		self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc")
		#apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan")
		apFile.removeFile("rmeasure.mrc")
	def createTemplateStack(self):
		"""
		takes the spider file and creates an average template of all particles
		"""

		templatestack = os.path.join(self.params['rundir'], "templatestack00.spi")
		apFile.removeFile(templatestack, warn=True)

		### hack to use standard filtering library
		templateparams = {}
		templateparams['apix'] = self.stack['apix']
		templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
		templateparams['templateIds'] = self.templatelist
		templateparams['bin'] = self.params['bin']
		templateparams['lowpass'] = self.params['lowpass']
		templateparams['median'] = None
		templateparams['pixlimit'] = None
		print templateparams
		apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
		filelist = apTemplate.getTemplates(templateparams)

		for mrcfile in filelist:
			emancmd  = ("proc2d templates/"+mrcfile+" "+templatestack
				+" clip="+str(self.boxsize)+","+str(self.boxsize)
				+" spider ")
			if self.params['inverttemplates'] is True:
				emancmd += " invert "
			apEMAN.executeEmanCmd(emancmd, showcmd=False)

		return templatestack
Esempio n. 40
0
    def makeEvenOddClasses(self, listfile, outputstack, classdata, maskrad):
        f = open(listfile, 'r')
        f.readline()
        lines = f.readlines()
        f.close()
        randstr = str(int(random.random() * 10e5))
        evenfile = self.rootname + "-even.lst"
        evenf = open(evenfile, 'w')
        oddfile = self.rootname + "-odd.lst"
        oddf = open(oddfile, 'w')
        evenf.write("#LST\n")
        oddf.write("#LST\n")
        neven = 0
        nodd = 0
        for i in range(0, len(lines)):
            if i % 2:
                nodd += 1
                oddf.write(lines[i])
            else:
                neven += 1
                evenf.write(lines[i])
        evenf.close()
        oddf.close()

        if neven > 0:
            self.makeClassAverages(evenfile, self.params['evenstack'],
                                   classdata, maskrad)
        if nodd > 0:
            self.makeClassAverages(oddfile, self.params['oddstack'], classdata,
                                   maskrad)
        apFile.removeFile(evenfile)
        apFile.removeFile(oddfile)
	def createSpiderFile(self):
		"""
		takes the stack file and creates a spider file ready for processing
		"""
		emancmd  = "proc2d "
		if not os.path.isfile(self.stack['file']):
			apDisplay.printError("stackfile does not exist: "+self.stack['file'])
		emancmd += self.stack['file']+" "

		spiderstack = os.path.join(self.params['rundir'], "start.spi")
		apFile.removeFile(spiderstack, warn=True)
		emancmd += spiderstack+" "

		emancmd += "apix="+str(self.stack['apix'])+" "
		if self.params['lowpass'] > 0:
			emancmd += "lp="+str(self.params['lowpass'])+" "
		if self.params['highpass'] > 0:
			emancmd += "hp="+str(self.params['highpass'])+" "
		if self.params['bin'] > 1:
			clipboxsize = self.boxsize*self.params['bin']
			emancmd += "shrink="+str(self.params['bin'])+" "
			emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" "
		emancmd += "last="+str(self.params['numpart']-1)+" "
		emancmd += "spider edgenorm"
		starttime = time.time()
		apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
		apEMAN.executeEmanCmd(emancmd, verbose=True)
		apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
		return spiderstack
Esempio n. 42
0
    def createTemplateStack(self):
        """
		convert spider template stack
		"""

        templatestack = os.path.join(self.params['rundir'],
                                     "templatestack.spi")
        apFile.removeFile(templatestack, warn=True)

        ### hack to use standard filtering library
        templateparams = {}
        templateparams['apix'] = self.stack['apix']
        templateparams['rundir'] = os.path.join(self.params['rundir'],
                                                "templates")
        templateparams['templateIds'] = self.templatelist
        templateparams['bin'] = self.params['bin']
        templateparams['lowpass'] = self.params['lowpass']
        templateparams['median'] = None
        templateparams['pixlimit'] = None
        print 'Converting reference templates:\n', templateparams
        apParam.createDirectory(
            os.path.join(self.params['rundir'], "templates"))
        filelist = apTemplate.getTemplates(templateparams)

        localclip = self.clipsize / self.params['bin']
        for mrcfile in filelist:
            emancmd = ("proc2d templates/" + mrcfile + " " + templatestack +
                       " clip=" + str(localclip) + "," + str(localclip) +
                       " edgenorm spiderswap ")
            if self.params['inverttemplates'] is True:
                emancmd += " invert "
            apEMAN.executeEmanCmd(emancmd, showcmd=False)

        return templatestack
def ClCla(alignedstack, numpart=None, numclasses=40,
                factorlist=range(1,5), corandata="coran/corandata", dataext=".spi"):
        """
        this doesn't work
        """
        if alignedstack[-4:] == dataext:
                alignedstack = alignedstack[:-4]

        rundir = "cluster"
        classavg = rundir+"/"+("classavgstack%03d" % numclasses)
        classvar = rundir+"/"+("classvarstack%03d" % numclasses)
        apParam.createDirectory(rundir)
        for i in range(numclasses):
                apFile.removeFile(rundir+("/classdoc%04d" % (i+1))+dataext)
        apFile.removeFile(rundir+"/clusterdoc"+dataext)

        factorstr, factorkey = operations.intListToString(factorlist)

        ### do hierarchical clustering
        mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
        mySpider.toSpider(
                "CL CLA",
                corandata, # path to coran data
                rundir+"/clusterdoc",   #clusterdoc file
                factorstr, #factor numbers
                "5,8",
                "4",
                "2", # minimum number of particles per class
                "Y", rundir+"/dendrogram.ps",
                "Y", rundir+"/dendrogramdoc",
        )
        mySpider.close()
	def undoCTFCorrect(self, newstackroot):
		# At this point, the stack needs to be remade un-ctf-corrected, and possibly normalized and/or inverted		
		apDisplay.printWarning('Relion needs a stack without ctf correction. A new stack is being made....')
		
		# Gather all the makestack parameters
		totalpart             = self.originalStackData.numpart
		numpart               = totalpart if not self.params['last'] else min(self.params['last'],totalpart)
		stackpathname         = os.path.basename( self.originalStackData.path )
		newstackrunname       = self.params['runname']
		newstackrundir        = self.params['rundir']
		newstackimagicfile    = os.path.join(newstackrundir,'start.hed')
		presetname            = self.originalStackData.preset
		
		# binning is combination of the original binning of the stack and the preparation binnning
		bin               = self.originalStackData.bin * self.params['bin']
		unbinnedboxsize   = self.stack['boxsize'] * self.originalStackData.bin
		lowpasstext       = setArgText( 'lowpass', ( self.params['lowpass'], self.originalStackData.lowpass ), False)
		highpasstext      = setArgText( 'highpass', ( self.params['highpass'], self.originalStackData.highpass ), True)
		partlimittext     = setArgText('partlimit',(numpart,),False)
		xmipp_normtext    = setArgText('xmipp-normalize', (self.params['xmipp-norm'],), True)
		sessionid         = int(self.params['expid'])
		sessiondata       = apDatabase.getSessionDataFromSessionId(sessionid)
		sessionname       = sessiondata['name']
		projectid         = self.params['projectid']
		stackid           = self.originalStackData.stackid
		reversetext       = '--reverse' if self.originalStackData.reverse else ''
		defoctext         = '--defocpair' if self.originalStackData.defocpair else ''
		inverttext        = '--no-invert' if not self.invert else ''  

		# Build the makestack2 command
		cmd = "makestack2.py "
		cmd += (" --single=%s --fromstackid=%d %s %s %s %s %s %s "%
			(os.path.basename(newstackimagicfile),selectionid,stackid,lowpasstext,
			highpasstext,partlimittext,reversetext,defoctext,inverttext))
		cmd += (" --normalized %s --boxsize=%d --bin=%d "%
			(xmipp_normtext,unbinnedboxsize,bin))
		cmd += (" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "%
			(stackpathname,stackid,projectid))
		cmd += (" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d "%
			(presetname,newstackrunname,newstackrundir,sessionname,sessionid))					
		cmd += ("  --no-wait --no-commit --no-continue  --jobtype=makestack2 ")	
		
		# Run the command
		logfilepath = os.path.join(newstackrundir,'relionstackrun.log')
		returncode = self.runAppionScriptInSubprocess(cmd,logfilepath)
		if returncode > 0:
			apDisplay.printError('Error in Relion specific stack making')

		# Make sure our new stack params reflects the changes made
		# Use the same complex equation as in eman clip
		clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin'])
		self.stack['boxsize']         = clipsize / self.params['bin']
		self.stack['apix']            = self.stack['apix'] * self.params['bin']
		self.stack['file']            = newstackroot+'.hed'		
		
		# Clean up
		boxfiles = glob.glob("*.box")
		for boxfile in boxfiles:
			apFile.removeFile(boxfile)
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"):
        """
        inputs:
                aligned particle stack
                number of classes
                timestamp
                output directory
        output:
                class averages
                class variances
                dendrogram.png
        """
        if timestamp is None:
                timestamp = apParam.makeTimestamp()

        classavg = rundir+"/"+("classavgstack_%s_%03d" %  (timestamp, numclasses))
        classvar = rundir+"/"+("classvarstack_%s_%03d" %  (timestamp, numclasses))

        thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext)

        ### create class doc files
        mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
        mySpider.toSpider(
                "CL HE",
                thresh,
                spyder.fileFilter(dendrogramfile), # dendrogram doc file
                rundir+"/classdoc_"+timestamp+"_****", # class doc file
        )

        ### delete existing files
        sys.stderr.write("delete existing files")
        for dext in (".hed", ".img", dataext):
                apFile.removeFile(classavg+dext)
                apFile.removeFile(classvar+dext)
        print ""

        ### create class averages
        sys.stderr.write("create class averages")
        for i in range(classes):
                sys.stderr.write(".")
                classnum = i+1
                mySpider.toSpiderQuiet(
                        "AS R",
                        spyder.fileFilter(alignedstack)+"@******",
                        rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)),
                        "A",
                        (classavg+"@%04d" % (classnum)),
                        (classvar+"@%04d" % (classnum)),
                )
        mySpider.close()
        print ""

        ### convert to IMAGIC
        emancmd = "proc2d "+classavg+".spi "+classavg+".hed"
        apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
        emancmd = "proc2d "+classvar+".spi "+classvar+".hed"
        apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)

        return classavg,classvar
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"):
	"""
	inputs:
		aligned particle stack
		number of classes
		timestamp
		output directory
	output:
		class averages
		class variances
		dendrogram.png
	"""
	if timestamp is None:
		timestamp = apParam.makeTimestamp()

	classavg = rundir+"/"+("classavgstack_%s_%03d" %  (timestamp, numclasses))
	classvar = rundir+"/"+("classvarstack_%s_%03d" %  (timestamp, numclasses))

	thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext)

	### create class doc files
	mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
	mySpider.toSpider(
		"CL HE",
		thresh,
		spyder.fileFilter(dendrogramfile), # dendrogram doc file
		rundir+"/classdoc_"+timestamp+"_****", # class doc file
	)

	### delete existing files
	sys.stderr.write("delete existing files")
	for dext in (".hed", ".img", dataext):
		apFile.removeFile(classavg+dext)
		apFile.removeFile(classvar+dext)
	print ""

	### create class averages
	sys.stderr.write("create %i class averages"%classes)
	for i in range(classes):
		sys.stderr.write(".")
		classnum = i+1
		mySpider.toSpiderQuiet(
			"AS R",
			spyder.fileFilter(alignedstack)+"@******",
			rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)),
			"A",
			(classavg+"@%04d" % (classnum)),
			(classvar+"@%04d" % (classnum)),
		)
	mySpider.close()
	print ""

	### convert to IMAGIC
	emancmd = "proc2d "+classavg+".spi "+classavg+".hed"
	apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
	emancmd = "proc2d "+classvar+".spi "+classvar+".hed"
	apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)

	return classavg,classvar
        def runAffinityPropagation(self, alignedstack):
                ### Get initial correlation values
                ### this is really, really slow
                similarfile, simlist = self.fillSimilarityMatrix(alignedstack)

                ### Preference value stats
                preffile = self.setPreferences(simlist)

                ### run apcluster.exe program
                outfile = "clusters.out"
                apDisplay.printMsg("Run apcluster.exe program")
                apclusterexe = os.path.join(apParam.getAppionDirectory(), "bin/apcluster.exe")
                apFile.removeFile(outfile)
                clustercmd = apclusterexe+" "+similarfile+" "+preffile+" "+outfile
                clusttime = time.time()
                proc = subprocess.Popen(clustercmd, shell=True)
                proc.wait()
                apDisplay.printMsg("apCluster time: "+apDisplay.timeString(time.time()-clusttime))

                if not os.path.isfile(outfile):
                        apDisplay.printError("apCluster did not run")

                ### Parse apcluster output file: clusters.out
                apDisplay.printMsg("Parse apcluster output file: "+outfile)
                clustf = open(outfile, "r")
                ### each line is the particle and the number is the class
                partnum = 0
                classes = {}
                for line in clustf:
                        sline = line.strip()
                        if sline:
                                partnum += 1
                                classnum = int(sline)
                                if not classnum in classes:
                                        classes[classnum] = [partnum,]
                                else:
                                        classes[classnum].append(partnum)
                clustf.close()
                apDisplay.printMsg("Found %d classes"%(len(classes.keys())))

                ### Create class averages
                classavgdata = []
                classnames = classes.keys()
                classnames.sort()
                for classnum in classnames:
                        apDisplay.printMsg("Class %d, %d members"%(classnum, len(classes[classnum])))
                        #clsf = open('subcls%04d.lst'%(classnum), 'w')
                        #for partnum in classes[classnum]:
                        #       clsf.write("%d\n"%(partnum))
                        #clsf.close()
                        classdatalist = apImagicFile.readParticleListFromStack(alignedstack, classes[classnum], msg=False)
                        classdatarray = numpy.asarray(classdatalist)
                        classavgarray = classdatarray.mean(0)
                        #mrc.write(classavgarray, 'subcls%04d.mrc'%(classnum))
                        classavgdata.append(classavgarray)
                apFile.removeStack("classaverage-"+self.timestamp+".hed")
                apImagicFile.writeImagic(classavgdata, "classaverage-"+self.timestamp+".hed")

                return classes
 def loopCleanUp(self, imgdata):
     ### last remove any existing boxed files, reset global params
     shortname = apDisplay.short(imgdata['filename'])
     shortfileroot = os.path.join(self.params['rundir'], shortname)
     rmfiles = glob.glob(shortfileroot + "*")
     if not self.params['keepall']:
         for rmfile in rmfiles:
             apFile.removeFile(rmfile)
Esempio n. 49
0
def emanMrcToStack(partlist):
    apFile.removeStack("emanmrc.hed", warn=False)
    for part in partlist:
        mrc.write(part, "temp.mrc")
        emancmd = "proc2d temp.mrc emanmrc.hed"
        apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
        apFile.removeFile("temp.mrc")
    return
Esempio n. 50
0
def emanSpiderToStack(partlist):
    apFile.removeStack("emanspi.hed", warn=False)
    for part in partlist:
        spider.write(part, "temp.spi")
        emancmd = "proc2d temp.spi emanspi.hed"
        apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
        apFile.removeFile("temp.spi")
    return
    def initialBPRP(self, classnum, volfile, spiderstack, eulerfile, numpart,
                    pixrad):

        # file that stores the number of iteration for BPRP
        BPRPcount = os.path.join(self.params['rundir'], str(classnum),
                                 "numiter.spi")

        if (os.path.isfile(BPRPcount)):
            apDisplay.printMsg("BP RP counter file exists: " + BPRPcount +
                               "! File will be deleted.")
            apFile.removeFile(BPRPcount)

        BPRPlambda = 2e-5
        backproject.backprojectRP(spiderstack,
                                  eulerfile,
                                  volfile,
                                  pixrad=pixrad,
                                  classnum=classnum,
                                  lambDa=BPRPlambda,
                                  numpart=numpart)

        count = 0
        rounds = 0

        ### repeat BPRP for 100 times with different values of lambda or until BPRP manages to do 50 iterations
        while count < 50 and rounds < 100:
            if (os.path.isfile(BPRPcount)):
                bc = open(BPRPcount, "r")
                for line in bc.readlines():
                    value = line.split()
                    if value[0] == "1":
                        count = int(float(value[2]))
                        if count < 50:
                            apDisplay.printMsg(
                                "BPRP iteration is " + str(count) +
                                " (less than 50)... redoing BPRP")
                            bc.close()
                            apFile.removeFile(BPRPcount)
                            BPRPlambda = BPRPlambda / 2
                            backproject.backprojectRP(spiderstack,
                                                      eulerfile,
                                                      volfile,
                                                      pixrad=pixrad,
                                                      classnum=classnum,
                                                      lambDa=BPRPlambda,
                                                      numpart=numpart)
            else:
                apDisplay.printWarning("numiter is missing")
                continue
            rounds += 1

        ### print warning if BPRP reaches 100 rounds
        if rounds >= 100:
            apDisplay.printWarning(
                "BPRP attempted 100 times but iteration is still less than 50. Check BPRP params."
            )

        return