def backproject3F(stackfile, eulerdocfile, volfile, numpart, dataext=".spi"): """ inputs: stack, in spider format eulerdocfile outputs: volume """ ### setup starttime = time.time() stackfile = spyder.fileFilter(stackfile) eulerdocfile = spyder.fileFilter(eulerdocfile) volfile = spyder.fileFilter(volfile) if not os.path.isfile(stackfile+dataext): apDisplay.printError("stack file not found: "+stackfile+dataext) if not os.path.isfile(eulerdocfile+dataext): apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext) apFile.removeFile(volfile+dataext) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("BP 3F", stackfile+"@*****", #stack file "1-%d"%(numpart), #number of particles eulerdocfile, #angle doc file "*", #input symmetry file, '*' for skip volfile, #filename for volume ) mySpider.close() apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext=".spi"): """ project 3D volumes using given Euler angles """ starttime = time.time() volfile = spyder.fileFilter(volfile) eulerdocfile = spyder.fileFilter(eulerdocfile) projstackfile = spyder.fileFilter(projstackfile) if not os.path.isfile(volfile+dataext): apDisplay.printError("volume file not found: "+volfile+dataext) if not os.path.isfile(eulerdocfile+dataext): apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext) apFile.removeFile(projstackfile) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("PJ 3Q", volfile, #input vol file str(pixrad), #pixel radius "1-%d"%(numpart), #number of particles eulerdocfile, #Euler DOC file projstackfile+"@*****", #output projections ) mySpider.close() apDisplay.printColor("finished projections in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def backprojectCG(stackfile, eulerdocfile, volfile, numpart, pixrad, dataext=".spi"): """ inputs: stack, in spider format eulerdocfile outputs: volume """ ### setup starttime = time.time() stackfile = spyder.fileFilter(stackfile) eulerdocfile = spyder.fileFilter(eulerdocfile) volfile = spyder.fileFilter(volfile) if not os.path.isfile(stackfile+dataext): apDisplay.printError("stack file not found: "+stackfile+dataext) if not os.path.isfile(eulerdocfile+dataext): apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext) apFile.removeFile(volfile+dataext) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("BP CG", stackfile+"@*****", #stack file "1-%d"%(numpart), #number of particles str(pixrad), #particle radius eulerdocfile, #angle doc file "N", #has symmetry?, does not work volfile, #filename for volume "%.1e,%.1f" % (1.0e-5, 0.0), #error, chi^2 limits "%d,%d" % (25,1), #iterations, 1st derivative mode "2000", #lambda - higher=less sensitive to noise ) mySpider.close() apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['numrefs'] is None: apDisplay.printError("a number of classes was not provided") maxparticles = 500000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid']) self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" %(self.params['clipsize'],self.clipsize)) self.clipsize = self.params['clipsize'] if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) self.mpirun = self.checkMPI() if self.mpirun is None: apDisplay.printError("There is no MPI installed") if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() if self.params['nproc'] < 2: apDisplay.printError("Only the MPI version of CL2D is currently supported, must run with > 1 CPU")
def checkConflicts(self): if self.params['lowpass'] > 0: apDisplay.printWarning( "lowpass filter value greater than zero; will ignore") if self.params['highpass'] > 0: apDisplay.printWarning( "highpass filter value greater than zero; will ignore") self.params['highpass'] = None self.params['lowpass'] = None #if self.params['kfactor'] is not None and self.params['numslices'] is not None: # apDisplay.printError("only one of 'kfactor' or 'numslices' can be defined") if self.params[ 'numslices'] is not None and self.params['numslices'] >= 15: apDisplay.printError( "too many slices defined by numslices, should be more like 2-6" ) if self.params['sizerange'] is not None and self.params[ 'sizerange'] >= 1.95 * self.params['diam']: apDisplay.printError( "size range has be less than twice the diameter") ### get number of processors: nproc = apParam.getNumProcessors() if not self.params['nproc']: self.params['nproc'] = nproc elif nproc < self.params['nproc']: apDisplay.printWarning( "Limiting number of processors to the %i that are available" % nproc) self.params['nproc'] = nproc return
def checkConflicts(self): if self.params['thresh'] is None: apDisplay.printError("threshold was not defined") ### Check if we have templates if self.params['templateliststr'] is None: apDisplay.printError( "template list was not specified, e.g. --template-list=34,56,12" ) ### Parse template list oldtemplateids = self.params['templateliststr'].split(',') self.params['templateIds'] = [] for tid in oldtemplateids: templateid = abs(int(tid)) self.params['templateIds'].append(templateid) if self.params['templatemirrors'] is True: self.params['templateIds'].append(-1 * templateid) ### Check if we have ranges if self.params['rangeliststr'] is None: apDisplay.printError( "range not specified, please provide range in the order of templateIds" ) ### Check that numbers of ranges and templates are equal rangestrlist = self.params['rangeliststr'].split('x') if len(oldtemplateids) != len(rangestrlist): apDisplay.printError( "the number of templates and ranges do not match") ### Parse range list i = 0 for rangestr in rangestrlist: self.params['range' + str(i)] = rangestr angs = rangestr.split(",") if not len(angs) == 3: apDisplay.printError("the range is not defined correctly") self.params['startang' + str(i + 1)] = float(angs[0]) self.params['endang' + str(i + 1)] = float(angs[1]) self.params['incrang' + str(i + 1)] = float(angs[2]) self.params['mirror' + str(i + 1)] = False if self.params['templatemirrors'] is True: i += 1 self.params['startang' + str(i + 1)] = float(angs[0]) self.params['endang' + str(i + 1)] = float(angs[1]) self.params['incrang' + str(i + 1)] = float(angs[2]) self.params['mirror' + str(i + 1)] = True i += 1 ### get number of processors: nproc = apParam.getNumProcessors() if not self.params['nproc']: self.params['nproc'] = nproc elif nproc < self.params['nproc']: apDisplay.printWarning( "Limiting number of processors to the %i that are available" % nproc) self.params['nproc'] = nproc return
def start(self): self.runtime = 0 self.checkCoranRun() ### convert stack to spider self.alignstackdata = self.getAlignedStack() maskpixrad = self.params['maskrad']/self.alignstackdata['pixelsize']/self.params['bin'] boxpixdiam = int(math.ceil(maskpixrad)+1)*2 if boxpixdiam*self.params['bin'] > self.alignstackdata['boxsize']: boxpixdiam = math.floor(self.alignstackdata['boxsize']/self.params['bin']) clippixdiam = boxpixdiam*self.params['bin'] apDisplay.printMsg("Pixel mask radius="+str(maskpixrad)) oldalignedstack = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['imagicfile']) alignedstackname = re.sub("\.", "_", self.alignstackdata['imagicfile'])+".spi" alignedstack = os.path.join(self.params['rundir'], alignedstackname) apFile.removeFile(alignedstack) emancmd = ("proc2d %s %s spiderswap shrink=%d clip=%d,%d edgenorm" %(oldalignedstack,alignedstack,self.params['bin'],clippixdiam,clippixdiam)) if self.params['numpart'] is not None: emancmd += " last=%d"%(self.params['numpart']-1) numpart = self.params['numpart'] else: numpart = self.getNumAlignedParticles() apEMAN.executeEmanCmd(emancmd, verbose=True) ### get number of processors nproc = apParam.getNumProcessors() esttime = classification.estimateTime(numpart, maskpixrad) apDisplay.printColor("Running spider this can take awhile, estimated time: "+\ apDisplay.timeString(esttime),"cyan") ### do correspondence analysis corantime = time.time() self.contriblist = classification.correspondenceAnalysis( alignedstack, boxsize=boxpixdiam, maskpixrad=maskpixrad, numpart=numpart, numfactors=self.params['numfactors'], nproc=nproc) corantime = time.time() - corantime ### make dendrogram dendrotime = time.time() classification.makeDendrogram(numfactors=min(3,self.params['numfactors']),nproc=nproc) dendrotime = time.time() - dendrotime inserttime = time.time() if self.params['commit'] is True: self.runtime = corantime self.insertCoranRun(insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apFile.removeFile(alignedstack, warn=True) apDisplay.printMsg("Correspondence Analysis time: "+apDisplay.timeString(corantime)) apDisplay.printMsg("Make Dendrogram time: "+apDisplay.timeString(dendrotime)) apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def checkConflicts(self): apDisplay.printMsg("/bin/rm -f *.hed *.img *.doc *.vol *.mrc *.spi *.log *~ *.pickle *.hist *.proj *.xmp *.sel *.basis *.original; rm -fr volume*") ### check for missing and duplicate entries #if self.params['alignid'] is None and self.params['clusterid'] is None: # apDisplay.printError("Please provide either --cluster-id or --align-id") if self.params['alignid'] is not None and self.params['clusterid'] is not None: apDisplay.printError("Please provide only one of either --cluster-id or --align-id") if self.params['modelstr'] is None and self.params['nvol'] is None: apDisplay.printError("Please provide model numbers or number of volumes") elif self.params['modelstr'] is not None: modellist = self.params['modelstr'].split(",") self.params['modelids'] = [] for modelid in modellist: self.params['modelids'].append(int(modelid)) ### get the stack ID from the other IDs if self.params['alignid'] is not None: self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignid']) self.params['stackid'] = self.alignstackdata['stack'].dbid elif self.params['clusterid'] is not None: self.clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid']) self.alignstackdata = self.clusterstackdata['clusterrun']['alignstack'] self.params['stackid'] = self.alignstackdata['stack'].dbid if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") #if self.params['description'] is None: # apDisplay.printError("run description was not defined") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if not self.params['fastmode'] in self.fastmodes: apDisplay.printError("fast mode must be on of: "+str(self.fastmodes)) maxparticles = 150000 minparticles = 50 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) if self.params['numpart'] < minparticles: apDisplay.printError("not enough particles requested, min: " + str(minparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) ### find number of processors if self.params['nproc'] is None: self.nproc = apParam.getNumProcessors() else: self.nproc = self.params['nproc'] self.mpirun = self.checkMPI()
def checkConflicts(self): if self.params["stackid"] is None: apDisplay.printError("stack id was not defined") if self.params["modelid"] is None: apDisplay.printError("model id was not defined") if self.params["runname"] is None: apDisplay.printError("missing a reconstruction name") if self.params["last"] is None: self.params["last"] = apStack.getNumberStackParticlesFromId(self.params["stackid"]) self.boxsize = apStack.getStackBoxsize(self.params["stackid"], msg=False) self.apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) if self.params["reconstackid"] is not None: reconboxsize = apStack.getStackBoxsize(self.params["reconstackid"], msg=False) reconapix = apStack.getStackPixelSizeFromStackId(self.params["reconstackid"]) refinenumpart = apStack.getNumberStackParticlesFromId(self.params["stackid"]) reconnumpart = apStack.getNumberStackParticlesFromId(self.params["reconstackid"]) if reconboxsize != self.boxsize: apDisplay.printError("Boxsize do not match for stacks") if reconapix != self.apix: apDisplay.printError("Pixelsize do not match for stacks") if refinenumpart != reconnumpart: apDisplay.printError("Number of particles do not match for stacks") maxmask = math.floor(self.apix * (self.boxsize - 10) / 2.0) if self.params["mask"] is None: apDisplay.printWarning("mask was not defined, setting to boxsize: %d" % (maxmask)) self.params["mask"] = maxmask if self.params["mask"] > maxmask: apDisplay.printWarning("mask was too big, setting to boxsize: %d" % (maxmask)) self.params["mask"] = maxmask if self.params["noctf"] is True: apDisplay.printWarning("Using no CTF method") self.params["wgh"] = -1.0 if self.params["nodes"] > 1 and self.params["cluster"] is False: apDisplay.printError("cluster mode must be enabled to run on more than 1 node") if self.params["ppn"] is None: if self.params["cluster"] is True: apDisplay.printError("you must define ppn for cluster mode") self.params["ppn"] = apParam.getNumProcessors() apDisplay.printMsg("Setting number of processors to %d" % (self.params["ppn"])) if self.params["rpn"] is None: self.params["rpn"] = self.params["ppn"] self.params["nproc"] = self.params["nodes"] * self.params["rpn"] ### get the symmetry data if self.params["sym"] is None: apDisplay.printError("Symmetry was not defined") else: self.symmdata = apSymmetry.findSymmetry(self.params["sym"]) self.params["symm_id"] = self.symmdata.dbid self.params["symm_name"] = self.symmdata["eman_name"] apDisplay.printMsg("Selected symmetry %s with id %s" % (self.symmdata["eman_name"], self.symmdata.dbid)) ### set cs value self.params["cs"] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['modelid'] is None: apDisplay.printError("model id was not defined") if self.params['runname'] is None: apDisplay.printError("missing a reconstruction name") if self.params['last'] is None: self.params['last'] = apStack.getNumberStackParticlesFromId(self.params['stackid']) self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False) self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) if self.params['reconstackid'] is not None: reconboxsize = apStack.getStackBoxsize(self.params['reconstackid'], msg=False) reconapix = apStack.getStackPixelSizeFromStackId(self.params['reconstackid']) refinenumpart = apStack.getNumberStackParticlesFromId(self.params['stackid']) reconnumpart = apStack.getNumberStackParticlesFromId(self.params['reconstackid']) if reconboxsize != self.boxsize: apDisplay.printError("Boxsize do not match for stacks") if reconapix != self.apix: apDisplay.printError("Pixelsize do not match for stacks") if refinenumpart != reconnumpart: apDisplay.printError("Number of particles do not match for stacks") maxmask = math.floor(self.apix*(self.boxsize-10)/2.0) if self.params['mask'] is None: apDisplay.printWarning("mask was not defined, setting to boxsize: %d"%(maxmask)) self.params['mask'] = maxmask if self.params['mask'] > maxmask: apDisplay.printWarning("mask was too big, setting to boxsize: %d"%(maxmask)) self.params['mask'] = maxmask if self.params['noctf'] is True: apDisplay.printWarning("Using no CTF method") self.params['wgh'] = -1.0 if self.params['nodes'] > 1 and self.params['cluster'] is False: apDisplay.printError("cluster mode must be enabled to run on more than 1 node") if self.params['ppn'] is None: if self.params['cluster'] is True: apDisplay.printError("you must define ppn for cluster mode") self.params['ppn'] = apParam.getNumProcessors() apDisplay.printMsg("Setting number of processors to %d"%(self.params['ppn'])) if self.params['rpn'] is None: self.params['rpn'] = self.params['ppn'] self.params['nproc'] = self.params['nodes']*self.params['rpn'] ### get the symmetry data if self.params['sym'] is None: apDisplay.printError("Symmetry was not defined") else: self.symmdata = apSymmetry.findSymmetry(self.params['sym']) self.params['symm_id'] = self.symmdata.dbid self.params['symm_name'] = self.symmdata['eman_name'] apDisplay.printMsg("Selected symmetry %s with id %s"%(self.symmdata['eman_name'], self.symmdata.dbid)) ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self): if self.params["thresh"] is None: apDisplay.printError("threshold was not defined") ### Check if we have templates if self.params["templateliststr"] is None: apDisplay.printError("template list was not specified, e.g. --template-list=34,56,12") ### Parse template list oldtemplateids = self.params["templateliststr"].split(",") self.params["templateIds"] = [] for tid in oldtemplateids: templateid = abs(int(tid)) self.params["templateIds"].append(templateid) if self.params["templatemirrors"] is True: self.params["templateIds"].append(-1 * templateid) ### Check if we have ranges if self.params["rangeliststr"] is None: apDisplay.printError("range not specified, please provide range in the order of templateIds") ### Check that numbers of ranges and templates are equal rangestrlist = self.params["rangeliststr"].split("x") if len(oldtemplateids) != len(rangestrlist): apDisplay.printError("the number of templates and ranges do not match") ### Parse range list i = 0 for rangestr in rangestrlist: self.params["range" + str(i)] = rangestr angs = rangestr.split(",") if not len(angs) == 3: apDisplay.printError("the range is not defined correctly") self.params["startang" + str(i + 1)] = float(angs[0]) self.params["endang" + str(i + 1)] = float(angs[1]) self.params["incrang" + str(i + 1)] = float(angs[2]) self.params["mirror" + str(i + 1)] = False if self.params["templatemirrors"] is True: i += 1 self.params["startang" + str(i + 1)] = float(angs[0]) self.params["endang" + str(i + 1)] = float(angs[1]) self.params["incrang" + str(i + 1)] = float(angs[2]) self.params["mirror" + str(i + 1)] = True i += 1 ### get number of processors: nproc = apParam.getNumProcessors() if not self.params["nproc"]: self.params["nproc"] = nproc elif nproc < self.params["nproc"]: apDisplay.printWarning("Limiting number of processors to the %i that are available" % nproc) self.params["nproc"] = nproc return
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['numrefs'] is None: apDisplay.printError("a number of classes was not provided") maxparticles = 500000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) # check for virtual stack self.params['virtualdata'] = None if not os.path.isfile(stackfile): vstackdata = apStack.getVirtualStackParticlesFromId( self.params['stackid']) npart = len(vstackdata['particles']) self.params['virtualdata'] = vstackdata else: npart = apFile.numImagesInStack(stackfile) if self.params['numpart'] > npart: apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid']) self.clipsize = int(math.floor( boxsize / float(self.params['bin'] * 2))) * 2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" % (self.params['clipsize'], self.clipsize)) self.clipsize = self.params['clipsize'] if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) self.mpirun = self.checkMPI() if self.mpirun is None: apDisplay.printError("There is no MPI installed") if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() if self.params['nproc'] < 2: apDisplay.printError( "Only the MPI version of CL2D is currently supported, must run with > 1 CPU" )
def start(self): """ This is the core of your function. You decide what happens here! """ stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp + ".hed") apFile.removeStack(filtered_stackfile, warn=False) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) boxsize = apStack.getStackBoxsize(self.params['stackid']) emancmd = "proc2d %s %s apix=%.3f " % (original_stackfile, filtered_stackfile, apix) if self.params['lowpass'] is not None: emancmd += " lp=%.3f " % (self.params['lowpass']) if self.params['highpass'] is not None: emancmd += " hp=%.3f " % (self.params['highpass']) if self.params['bin'] is not None and self.params['bin'] > 1: ## determine a multiple of the bin that is divisible by 2 and less than the boxsize clipsize = int(math.floor(boxsize / float( self.params['bin'] * 2))) * 2 * self.params['bin'] emancmd += " shrink=%d clip=%d,%d " % (self.params['bin'], clipsize, clipsize) emancmd += " last=%d " % (self.params['numpart'] - 1) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) ### confirm that it worked if self.params['numpart'] != apFile.numImagesInStack( filtered_stackfile): apDisplay.printError("Missing particles in stack") nproc = apParam.getNumProcessors() ### run the refine2d.py emancmd = ("refine2d.py --iter=%d --ninitcls=%d --proc=%d %s" % ( self.params['numiter'], self.params['numclasses'], nproc, filtered_stackfile, )) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) self.commitToDatabase()
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['description'] is None: apDisplay.printError("run description was not defined") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if self.params['partrad'] is None: apDisplay.printError("particle radius was not defined") ### deal with particles maxparticles = 150000 self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile))) ### get num processors if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() ### convert / check template data if self.params['templatelist'] is None: apDisplay.printError("template list was not provided") self.templatelist = self.params['templatelist'].strip().split(",") if not self.templatelist or type(self.templatelist) != type([]): apDisplay.printError("could not parse template list=" + self.params['templatelist']) self.params['numtemplate'] = len(self.templatelist) apDisplay.printMsg("Found " + str(self.params['numtemplate']) + " templates") ### convert / check template data if self.params['orientref'] is None: apDisplay.printError("reference for orientation was not provided")
def fermiLowPassFilter(imgarray, pixrad=2.0, dataext="spi", nproc=None): if dataext[0] == '.': dataext = dataext[1:] if nproc is None: nproc = apParam.getNumProcessors(msg=False) ### save array to spider file spider.write(imgarray, "rawimg."+dataext) ### run the filter mySpider = spyder.SpiderSession(dataext=dataext, logo=False, nproc=nproc) ### filter request: infile, outfile, filter-type, inv-radius, temperature mySpider.toSpiderQuiet("FQ", "rawimg", "filtimg", "5", str(1.0/pixrad), "0.04") mySpider.close() ### read array from spider file filtarray = spider.read("filtimg."+dataext) ### clean up apFile.removeFile("rawimg."+dataext) apFile.removeFile("filtimg."+dataext) return filtarray
def backprojectRP(stackfile, eulerdocfile, volfile, pixrad, classnum, lambDa, numpart=None, selfile=None, dataext=".spi"): """ inputs: stack, in spider format eulerdocfile outputs: volume """ ### setup starttime = time.time() stackfile = spyder.fileFilter(stackfile) eulerdocfile = spyder.fileFilter(eulerdocfile) if selfile is not None: selfile = spyder.fileFilter(selfile) volfile = spyder.fileFilter(volfile) counter = spyder.fileFilter("numiter") if not os.path.isfile(stackfile+dataext): apDisplay.printError("stack file not found: "+stackfile+dataext) if not os.path.isfile(eulerdocfile+dataext): apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext) apFile.removeFile(volfile+dataext) if (numpart is not None): selection = "1-"+str(numpart) elif (selfile is not None): selection = selfile else: apDisplay.printError("Partilce selection is invalid for BP RP. Please make sure either numpart or selfile is specified in the function backprojectRP!") nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("BP RP x11", stackfile+"@*****", #stack file selection, #selection file or particle range str(pixrad), #particle radius eulerdocfile, #angle doc file "*", #has symmetry?, does not work volfile, #filename for volume "%.1e,%.1f" % (lambDa, 0.0), #lambda, correction "%d,%d" % (50,1), #iteration limit, mode "%d,%d" % (0,0), #minimum, maximum "%.5f" % ((1/(1+6*lambDa))*0.999), #smoothing constant = (1/(1+6*lambda))*0.999 ) mySpider.toSpider("SD 1,x11", str(classnum)+"/"+str(counter)) mySpider.close() apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def fermiLowPassFilter(imgarray, pixrad=2.0, dataext="spi", nproc=None): if dataext[0] == '.': dataext = dataext[1:] if nproc is None: nproc = apParam.getNumProcessors(msg=False) ### save array to spider file spider.write(imgarray, "rawimg." + dataext) ### run the filter mySpider = spyder.SpiderSession(dataext=dataext, logo=False, nproc=nproc) ### filter request: infile, outfile, filter-type, inv-radius, temperature mySpider.toSpiderQuiet("FQ", "rawimg", "filtimg", "5", str(1.0 / pixrad), "0.04") mySpider.close() ### read array from spider file filtarray = spider.read("filtimg." + dataext) ### clean up apFile.removeFile("rawimg." + dataext) apFile.removeFile("filtimg." + dataext) return filtarray
def estimateIterTime(self): secperiter = 0.12037 ### get num processors if self.params['nproc'] is None: nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] calctime = ((self.params['numpart'] / 1000.0) * self.params['numtemplate'] * (self.stack['boxsize'] / self.params['bin'])**2 / self.params['psistep'] / float(nproc) * secperiter) if self.params['mirror'] is True: calctime *= 2.0 self.params['estimatedtime'] = calctime apDisplay.printColor( "Estimated first iteration time: " + apDisplay.timeString(calctime), "purple")
def estimateIterTime(self): secperiter = 0.12037 ### get num processors if self.params['nproc'] is None: nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] calctime = ( (self.params['numpart']/1000.0) *self.params['numrefs'] *(self.stack['boxsize']/self.params['bin'])**2 /self.params['psistep'] /float(nproc) *secperiter ) if self.params['mirror'] is True: calctime *= 2.0 self.params['estimatedtime'] = calctime apDisplay.printColor("Estimated first iteration time: "+apDisplay.timeString(calctime), "purple")
def threadCommands(commandlist, nproc=None, pausetime=1.0): """ takes a list of commands and runs N of them at a time waits for one to finish and submits another thread caveats: only work on a single machine """ ### set number of processes at a time if nproc is None: nproc = apParam.getNumProcessors() if len(commandlist) < nproc: nproc = len(commandlist) ### copy the list so we don't modify the original localcmdlist = commandlist joblist = [] ### initialize threads for i in range(nproc): cmd = localcmdlist.pop(0) writeThreadLog(cmd) job = AppionJob(cmd) joblist.append(job) job.run() ### continue until no commands are left sys.stderr.write("threading commands") while len(localcmdlist) > 0 or len(joblist) > 0: for job in joblist: if job.poll() is not None: joblist.remove(job) if len(localcmdlist) > 0: cmd = localcmdlist.pop(0) writeThreadLog(cmd) job = AppionJob(cmd) joblist.append(job) job.run() sys.stderr.write(".") time.sleep(pausetime) #print joblist sys.stderr.write("\n")
def checkConflicts(self): if self.params['lowpass'] > 0: apDisplay.printWarning("lowpass filter value greater than zero; will ignore") if self.params['highpass'] > 0: apDisplay.printWarning("highpass filter value greater than zero; will ignore") self.params['highpass'] = None self.params['lowpass'] = None #if self.params['kfactor'] is not None and self.params['numslices'] is not None: # apDisplay.printError("only one of 'kfactor' or 'numslices' can be defined") if self.params['numslices'] is not None and self.params['numslices'] >= 15: apDisplay.printError("too many slices defined by numslices, should be more like 2-6") if self.params['sizerange'] is not None and self.params['sizerange'] >= 1.95*self.params['diam']: apDisplay.printError("size range has be less than twice the diameter") ### get number of processors: nproc = apParam.getNumProcessors() if not self.params['nproc']: self.params['nproc'] = nproc elif nproc < self.params['nproc']: apDisplay.printWarning("Limiting number of processors to the %i that are available"%nproc) self.params['nproc'] = nproc return
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['description'] is None: apDisplay.printError("run description was not defined") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if self.params['partrad'] is None: apDisplay.printError("particle radius was not defined") ### deal with particles maxparticles = 150000 self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: "+str(maxparticles)+" requested: "+str(self.params['numpart'])) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) ### get num processors if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() ### convert / check template data if self.params['templatelist'] is None: apDisplay.printError("template list was not provided") self.templatelist = self.params['templatelist'].strip().split(",") if not self.templatelist or type(self.templatelist) != type([]): apDisplay.printError("could not parse template list="+self.params['templatelist']) self.params['numtemplate'] = len(self.templatelist) apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates") ### convert / check template data if self.params['orientref'] is None: apDisplay.printError("reference for orientation was not provided")
def start(self): """ This is the core of your function. You decide what happens here! """ stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp+".hed") apFile.removeStack(filtered_stackfile, warn=False) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) boxsize = apStack.getStackBoxsize(self.params['stackid']) emancmd = "proc2d %s %s apix=%.3f "%(original_stackfile, filtered_stackfile, apix) if self.params['lowpass'] is not None: emancmd += " lp=%.3f "%(self.params['lowpass']) if self.params['highpass'] is not None: emancmd += " hp=%.3f "%(self.params['highpass']) if self.params['bin'] is not None and self.params['bin'] > 1: ## determine a multiple of the bin that is divisible by 2 and less than the boxsize clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2*self.params['bin'] emancmd += " shrink=%d clip=%d,%d "%(self.params['bin'], clipsize, clipsize) emancmd += " last=%d "%(self.params['numpart']-1) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) ### confirm that it worked if self.params['numpart'] != apFile.numImagesInStack(filtered_stackfile): apDisplay.printError("Missing particles in stack") nproc = apParam.getNumProcessors() ### run the refine2d.py emancmd = ("refine2d.py --iter=%d --ninitcls=%d --proc=%d %s" %(self.params['numiter'], self.params['numclasses'], nproc, filtered_stackfile,) ) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) self.commitToDatabase()
def checkConflicts(self): """ basic error handlign for running scripts """ ### check for IMAGIC installation try: self.imagicroot = apIMAGIC.checkImagicExecutablePath() except: c = "some IMAGIC functions, e.g. prealignment and angular reconstitution " c+= "may not work. Use EMAN instead" apDisplay.printWarning(c) self.imagicroot = None self.params['useEMAN1'] is True ### check class averages if self.params['templatestackid'] is not None: self.stackdata = appiondata.ApTemplateStackData.direct_query(self.params['templatestackid']) self.clsname = self.stackdata['templatename'] self.params['apix'] = self.stackdata['apix'] self.params['boxsize'] = self.stackdata['boxsize'] self.params['oldavgs'] = os.path.join(self.stackdata['path']['path'], self.clsname[:-4]+".hed") self.params['avgs'] = os.path.basename(self.clsname) elif self.params['clusterid'] is not None: self.stackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid']) self.clsname = self.stackdata['avg_imagicfile'] self.params['apix'] = self.stackdata['clusterrun']['pixelsize'] self.params['boxsize'] = self.stackdata['clusterrun']['boxsize'] self.params['oldavgs'] = os.path.join(self.stackdata['path']['path'], self.clsname[:-4]+".hed") self.params['avgs'] = self.clsname elif self.params['classavgs'] is not None: if self.params['apix'] is None: apDisplay.printError("enter pixel size for manually uploaded classes") self.params['oldavgs'] = self.params['classavgs'] bavgs = os.path.basename(self.params['classavgs']) self.params['avgs'] = bavgs else: apDisplay.printError("enter class averages for the run") if not os.path.isfile(os.path.abspath(self.params['oldavgs'])): apDisplay.printError("cannot find input class averages") if self.params['templatestackid'] is not None and self.params['clusterid'] is not None: apDisplay.printError("enter either templatestack ID OR cluster ID") if self.params['templatestackid'] is not None and self.params['classavgs'] is not None: apDisplay.printError("enter either templatestack ID OR manually uploaded classes") if self.params['clusterid'] is not None and self.params['classavgs'] is not None: apDisplay.printError("enter either cluster ID OR manually uploaded classes") ### pretreatment if self.imagicroot is not None and self.params['prealign'] is True: warning = "particles will not be prealigned, IMAGIC is not installed or path " warning+= "to IMAGIC cannot be located" apDisplay.printWarning(warning) ### basic input parameters self.params['numpart'] = apFile.numImagesInStack(self.params['oldavgs']) if self.params['numpart'] < 3: apDisplay.printError("need at least 3 class averages in stack to run") self.params['boxsize'] = apFile.getBoxSize(self.params['oldavgs'])[0] if self.params['num_volumes'] is None: apDisplay.printError("specify the number of volumes to produce") if self.params['apix'] is None: apDisplay.printError("enter pixel size of class averages") ### refinement class averages if self.params['r_templatestackid'] is not None: self.rstackdata = appiondata.ApTemplateStackData.direct_query(self.params['r_templatestackid']) self.rclsname = self.rstackdata['templatename'] self.params['refineapix'] = self.rstackdata['apix'] self.params['refineboxsize'] = self.rstackdata['boxsize'] self.params['oldravgs'] = os.path.join(self.rstackdata['path']['path'], self.rclsname[:-4]+".hed") self.params['ravgs'] = os.path.basename(self.rclsname) elif self.params['r_clusterid'] is not None: self.rstackdata = appiondata.ApClusteringStackData.direct_query(self.params['r_clusterid']) self.rclsname = self.rstackdata['avg_imagicfile'] self.params['refineapix'] = self.rstackdata['clusterrun']['pixelsize'] self.params['refineboxsize'] = self.rstackdata['clusterrun']['boxsize'] self.params['oldravgs'] = os.path.join(self.rstackdata['path']['path'], self.rclsname[:-4]+".hed") self.params['ravgs'] = self.rclsname elif self.params['refine_classavgs'] is not None: rbavgs = os.path.basename(self.params['refine_classavgs']) self.params['ravgs'] = rbavgs self.params['refineboxsize'] = apFile.getBoxSize(self.params['oldravgs'])[0] if self.params['refineapix'] is None: warning = "refinement pixel size not specified ... assuming it is the same" apDisplay.printWarning(warning) self.params['refineapix'] = self.params['apix'] else: # self.params['ravgs'] = None self.params['oldravgs'] = self.params['avgs'] self.params['ravgs'] = self.params['avgs'] self.params['refineapix'] = self.params['apix'] self.params['refineboxsize'] = self.params['boxsize'] if not os.path.isfile(os.path.abspath(self.params['oldravgs'])): warning = "cannot find class averages for refinement using the " warning+= "specified path. Original averages will be used instead." apDisplay.printWarning(warning) self.params['oldravgs'] = self.params['avgs'] self.params['ravgs'] = self.params['avgs'] ### check for scaling if self.params['scale'] is True: self.scalefactor = float(64.0 / self.params['boxsize']) self.params['apix'] = self.params['apix'] / self.scalefactor self.params['boxsize'] = 64 else: self.scalefactor = 1 ### angular reconstitution checks if self.params['keep_ordered'] < 1.0: ### probably specified as a fraction self.params['keep_ordered'] = self.params['keep_ordered'] * 100 ### convert to percentage self.params['keep_ordered_num'] = self.params['numpart'] * self.params['keep_ordered'] / 100 ### EMAN1 cross common lines checks if self.params['useEMAN1'] is True: if self.params['images_per_volume'] is None: self.params['images_per_volume'] = int(self.params['numpart'] * 0.66) if self.params['images_per_volume'] > self.params['numpart']: warning = "number of images per volume greater than number of class averages " warning+= "in input. Setting to (0.66)*(num_averages)" apDisplay.printWarning(warning) self.params['images_per_volume'] = int(self.params['numpart'] * 0.66) if self.params['images_per_volume'] > self.params['numpart'] * 0.75: warning = "consider asking for less images per volume to provide more variety " warning = "during common lines search" apDisplay.printWarning(warning) ### number of processors for threading ONLY works on a single node self.params['threadnproc'] = apParam.getNumProcessors() ### refinement parameters if self.params['linmask'] == 0: self.params['linmask'] = (self.params['boxsize']-2)/2*self.params['apix'] if self.params['mask_radius'] is None and self.params['linmask'] != 0: self.params['mask_radius'] = self.params['linmask'] * 1.2 elif self.params['mask_radius'] is None and self.params['linmask'] == 0: self.params['mask_radius'] = self.params['boxsize'] * self.params['refineapix'] if self.params['outer_radius'] is None: self.params['outer_radius'] = self.params['mask_radius'] * 0.8 return
def start(self): if self.params['method'] == 'any': ### startAny uses class averages clusterstack, numimages = self.getClusterStack() else: ### starticos, startoct, startcsym uses individual particles clusterstack, numimages = self.getClusterParticles() if self.params['method'] != 'any': if self.params['numkeep'] is not None and numimages/10 < int(self.params['numkeep']): apDisplay.printWarning("particle number of "+ self.params['numkeep'] + " is greater than 10% of the number of selected classes") elif self.params['numkeep'] is None: self.params['numkeep'] = int(math.floor(numimages/20.0))+1 apDisplay.printWarning("numkeep was not defined, using %d particles"%(self.params['numkeep'])) nproc = apParam.getNumProcessors() #construct command for each of the EMAN commonline method if self.params['method'] == 'any': startcmd = "startAny "+clusterstack+" proc="+str(nproc) startcmd +=" sym="+self.symmdata['eman_name'] if self.params['mask'] is not None: startcmd +=" mask="+str(self.params['mask']) else: maskrad = math.floor(self.clusterstackdata['clusterrun']['boxsize']/2.0) startcmd +=" mask=%d"%(maskrad) if self.params['rounds'] is not None: startcmd +=" rounds="+str(self.params['rounds']) elif self.params['method'] == 'csym': startcmd = "startcsym "+clusterstack+" " startcmd +=" "+str(self.params['numkeep']) startcmd +=" sym="+self.symmdata['eman_name'] if self.params['imask'] is not None: startcmd +=" imask="+self.params['imask'] elif self.params['method'] == 'oct': startcmd = "startoct "+clusterstack+" " startcmd +=" "+str(self.params['numkeep']) elif self.params['method'] == 'icos': startcmd = "starticos "+clusterstack+" " startcmd +=" "+str(self.params['numkeep']) if self.params['imask'] is not None: startcmd +=" imask="+self.params['imask'] apDisplay.printMsg("Creating 3D model with EMAN function: start"+self.params['method']) apFile.removeFile("threed.0a.mrc") apFile.removeFile("eman.log") apEMAN.executeEmanCmd(startcmd, verbose=False, logfile="eman.log") #apEMAN.executeEmanCmd(startcmd, verbose=True) finalmodelname = "threed-%s-eman_start%s.mrc"%(self.timestamp, self.params['method']) finalmodelpath = os.path.join(self.params['rundir'], finalmodelname) apDisplay.printMsg("Final model name: "+finalmodelname) finalmodel = "threed.0a.mrc" if os.path.isfile(finalmodel): emancmd = "proc3d %s %s norm=0,1 origin=0,0,0"%(finalmodel, finalmodelpath) #shutil.move(finalmodel, finalmodelpath) apEMAN.executeEmanCmd(emancmd, verbose=True) if not apVolume.isValidVolume(finalmodelpath): apDisplay.printError("Created volume is not valid") else: apDisplay.printError("No 3d model was created") ### upload it self.uploadDensity(finalmodelpath) ### chimera imaging apChimera.renderSnapshots(finalmodelpath, contour=self.params['contour'], zoom=self.params['zoom'], sym=self.symmdata['eman_name']) apChimera.renderAnimation(finalmodelpath, contour=self.params['contour'], zoom=self.params['zoom'], sym=self.symmdata['eman_name']) ### remove stack if apFile.stackSize(clusterstack)/1024**2 > 10: ### file bigger than 10MB apFile.removeStack(clusterstack) apFile.removeFile("threed.0a.mrc")
def alignStack(oldstack, alignedstack, partlist, dataext=".spi"): """ write aligned stack -- with python loop inputs: oldstack newstack (empty) list of particle dictionaries for operations modifies: newstack output: none I tried this loop in both spider and python; python was faster?!? -neil """ if not os.path.isfile(oldstack + dataext): apDisplay.printError("Could not find original stack: " + oldstack + dataext) boxsize = apFile.getBoxSize(oldstack + dataext) apDisplay.printMsg("applying alignment parameters to stack") apFile.removeFile(alignedstack + dataext) count = 0 t0 = time.time() nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) #create stack in core numpart = len(partlist) mySpider.toSpiderQuiet( "MS I", #command "_2@", #name "%d,%d,%d" % (boxsize), #boxsize str(numpart + 1), #num part to create in memory str(numpart + 1), #max particle number ) for partdict in partlist: partnum = partdict['num'] #if partdict['num'] in [3,6,7]: # print partdict['num'], partdict['template'], partdict['mirror'], round(partdict['rot'],3) ### Rotate and Shift operations count += 1 #rotate/shift mySpider.toSpiderQuiet( "RT SQ", spyder.fileFilter(oldstack) + "@" + ("%06d" % (partnum)), "_1", str(partdict['rot']), str(partdict['xshift']) + "," + str(partdict['yshift']), ) #mirror, if necessary if 'mirror' in partdict and partdict['mirror'] is True: mySpider.toSpiderQuiet( "MR", "_1", "_2@" + ("%06d" % (partnum)), "Y", ) else: mySpider.toSpiderQuiet( "CP", "_1", "_2@" + ("%06d" % (partnum)), ) ### finish up #save stack to file mySpider.toSpiderQuiet( "CP", "_2@", spyder.fileFilter(alignedstack) + "@", ) #delete stack mySpider.toSpiderQuiet( "DE", "_2", ) mySpider.close() apDisplay.printMsg("Completed transforming %d particles in %s" % (count, apDisplay.timeString(time.time() - t0))) if count < 1: apDisplay.printError("Failed to transform any particles") if not os.path.isfile(alignedstack + dataext): apDisplay.printError("Failed to create stack " + alignedstack + dataext) return
def start(self): self.insertMaxLikeJob() self.stack = {} self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) if self.params['virtualdata'] is not None: self.stack['file'] = self.params['virtualdata']['filename'] else: self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) self.estimateIterTime() self.dumpParameters() ### process stack to local file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") a = proc2dLib.RunProc2d() a.setValue('infile',self.stack['file']) a.setValue('outfile',self.params['localstack']) a.setValue('apix',self.stack['apix']) a.setValue('bin',self.params['bin']) a.setValue('last',self.params['numpart']-1) a.setValue('append',False) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: a.setValue('lowpass',self.params['lowpass']) if self.params['highpass'] is not None and self.params['highpass'] > 1: a.setValue('highpass',self.params['highpass']) if self.params['invert'] is True: a.setValue('invert') is True if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) # clip not yet implemented # self.params['clipsize'] is not None: # clipsize = int(self.clipsize)*self.params['bin'] # if clipsize % 2 == 1: # clipsize += 1 ### making sure that clipped boxsize is even # a.setValue('clip',clipsize) if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) #run proc2d a.run() if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']): apDisplay.printError("Missing particles in stack") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) ### stack is needed by uploadMaxlike.py #apFile.removeStack(self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " +" -i "+os.path.join(self.params['rundir'], self.partlistdocfile) +" -nref "+str(self.params['numrefs']) +" -iter "+str(self.params['maxiter']) +" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp) +" -psi_step "+str(self.params['psistep']) ) ### fast mode if self.params['fast'] is True: xmippopts += " -fast " if self.params['fastmode'] == "narrow": xmippopts += " -C 1e-10 " elif self.params['fastmode'] == "wide": xmippopts += " -C 1e-17 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " ### normalization if self.params['norm'] is True: xmippopts += " -norm " ### use student's T distribution if self.params['student'] is True: xmippopts += " -student " ### write cluster job file if self.params['cluster'] is True: self.writeClusterJobFile() ### find number of processors if self.params['nproc'] is None: nproc = nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] mpirun = self.checkMPI() self.estimateIterTime() if nproc > 2 and mpirun is not None: ### use multi-processor apDisplay.printColor("Using "+str(nproc)+" processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True) mpiruncmd = mpirun+" -np "+str(nproc)+" "+xmippexe+" "+xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True) else: ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe+" "+xmippopts self.writeXmippLog(xmippcmd) apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ### minor post-processing self.createReferenceStack() self.readyUploadFlag() self.dumpParameters()
def checkConflicts(self): ### first get all stack data self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### modify boxsize and pixelsize according to binning self.boxsize = self.stack['boxsize'] self.clipsize = int(math.floor(self.boxsize/float(self.params['bin']*2)))*2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" %(self.params['clipsize'],self.clipsize)) self.clipsize = self.params['clipsize'] self.apix = self.stack['apix'] if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize)*self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even self.boxsize = clipsize self.apix = self.apix * self.params['bin'] ### basic error checking if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['ncls'] is None: apDisplay.printError("a number of classes was not provided") maxparticles = 1000000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) if self.params['numpart'] > 5000: apDisplay.printWarning("initial model calculation may not work with less than 5000 particles") self.mpirun = self.checkMPI() # if self.mpirun is None: # apDisplay.printError("There is no MPI installed") if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() ### SIMPLE defaults and error checking if self.params['ring2'] is None: self.params['ring2'] = (self.boxsize/2) - 2 if self.params['ncls'] > 2000: apDisplay.printError("number of classes should be less than 2000 for subsequent ORIGAMI run to work") if self.params['ncls'] > self.params['numpart']: self.params['ncls'] = self.params['numpart'] / self.params['minp'] if self.params['mask'] is None: self.params['mask'] = (self.boxsize/2) - 2 if self.params['mw'] is None: apDisplay.printError("please specify the molecular weight (in kDa)")
def uploadScriptData(self): """ Using tables to track program run parameters in a generic fashion inspired by Roberto Marabini and Carlos Oscar Sanchez Sorzano from the Xmipp team/Carazo lab """ apDisplay.printMsg("Uploading ScriptData....") prognameq = appiondata.ScriptProgramName() prognameq['name'] = self.functionname userq = appiondata.ScriptUserName() userdict = apParam.getUserDict() if userdict: userq['name'] = userdict['username'] userq['uid'] = userdict['uid'] userq['gid'] = userdict['gid'] userq['fullname'] = userdict['fullname'] unixshell = userdict['unixshell'] else: userq['name'] = "unknown" unixshell = None hostq = appiondata.ScriptHostName() hostq['name'] = apParam.getHostname() hostq['ip'] = apParam.getHostIP() hostq['system'] = apParam.getSystemName() hostq['distro'] = apParam.getLinuxDistro() hostq['nproc'] = apParam.getNumProcessors() hostq['memory'] = apParam.getTotalMemory() hostq['cpu_vendor'] = apParam.getCPUVendor() hostq['gpu_vendor'] = apParam.getGPUVendor() hostq['arch'] = apParam.getMachineArch() progrunq = appiondata.ScriptProgramRun() progrunq['runname'] = self.params['runname'] progrunq['progname'] = prognameq progrunq['username'] = userq progrunq['hostname'] = hostq progrunq['unixshell'] = unixshell progrunq['rundir'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) progrunq['job'] = self.getClusterJobData() appiondir = apParam.getAppionDirectory() ### get appion version/subversion revision progrunq['revision'] = None versionfile = os.path.join(appiondir, "appionlib/version.txt") if os.path.isfile(versionfile): f = open(versionfile, 'r') line = f.readline() f.close() sline = line.strip() progrunq['revision'] = sline if os.path.isdir(os.path.join(appiondir, ".svn")): if progrunq['revision'] is None: progrunq['revision'] = version.getSubversionRevision(appiondir) else: progrunq['revision'] += "-" + version.getSubversionRevision( appiondir) if not progrunq['revision']: progrunq['revision'] = 'unknown' apDisplay.printMsg("Running Appion version '%s'" % (progrunq['revision'])) progrunq['appion_path'] = appiondata.ApPathData( path=os.path.abspath(appiondir)) for paramname in self.params.keys(): paramnameq = appiondata.ScriptParamName() paramnameq['name'] = paramname paramnameq['progname'] = prognameq paramvalueq = appiondata.ScriptParamValue() paramvalueq['value'] = str(self.params[paramname]) usage = self.usageFromParamDest(paramname, self.params[paramname]) #print "usage: ", usage paramvalueq['usage'] = usage paramvalueq['paramname'] = paramnameq paramvalueq['progrun'] = progrunq if usage is not None: paramvalueq.insert()
def uploadScriptData(self): """ Using tables to track program run parameters in a generic fashion inspired by Roberto Marabini and Carlos Oscar Sanchez Sorzano from the Xmipp team/Carazo lab """ apDisplay.printMsg("Uploading ScriptData....") prognameq = appiondata.ScriptProgramName() prognameq['name'] = self.functionname userq = appiondata.ScriptUserName() userdict = apParam.getUserDict() if userdict: userq['name'] = userdict['username'] userq['uid'] = userdict['uid'] userq['gid'] = userdict['gid'] userq['fullname'] = userdict['fullname'] unixshell = userdict['unixshell'] else: userq['name'] = "unknown" unixshell = None hostq = appiondata.ScriptHostName() hostq['name'] = apParam.getHostname() hostq['ip'] = apParam.getHostIP() hostq['system'] = apParam.getSystemName() hostq['distro'] = apParam.getLinuxDistro() hostq['nproc'] = apParam.getNumProcessors() hostq['memory'] = apParam.getTotalMemory() hostq['cpu_vendor'] = apParam.getCPUVendor() hostq['gpu_vendor'] = apParam.getGPUVendor() hostq['arch'] = apParam.getMachineArch() progrunq = appiondata.ScriptProgramRun() progrunq['runname'] = self.params['runname'] progrunq['progname'] = prognameq progrunq['username'] = userq progrunq['hostname'] = hostq progrunq['unixshell'] = unixshell progrunq['rundir'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) progrunq['job'] = self.getClusterJobData() appiondir = apParam.getAppionDirectory() ### get appion version/subversion revision progrunq['revision'] = None versionfile = os.path.join(appiondir, "appionlib/version.txt") if os.path.isfile(versionfile): f = open(versionfile, 'r') line = f.readline() f.close() sline = line.strip() progrunq['revision'] = sline if os.path.isdir(os.path.join(appiondir, ".svn")): if progrunq['revision'] is None: progrunq['revision'] = version.getSubversionRevision(appiondir) else: progrunq['revision'] += "-"+version.getSubversionRevision(appiondir) if not progrunq['revision']: progrunq['revision'] = 'unknown' apDisplay.printMsg("Running Appion version '%s'"%(progrunq['revision'])) progrunq['appion_path'] = appiondata.ApPathData(path=os.path.abspath(appiondir)) for paramname in self.params.keys(): paramnameq = appiondata.ScriptParamName() paramnameq['name'] = paramname paramnameq['progname'] = prognameq paramvalueq = appiondata.ScriptParamValue() paramvalueq['value'] = str(self.params[paramname]) usage = self.usageFromParamDest(paramname, self.params[paramname]) #print "usage: ", usage paramvalueq['usage'] = usage paramvalueq['paramname'] = paramnameq paramvalueq['progrun'] = progrunq if usage is not None: paramvalueq.insert()
def start(self): self.insertMaxLikeJob() self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.estimateIterTime() self.dumpParameters() ### process stack to local temp file proccmd = "proc2d "+self.stack['file']+" temp.hed apix="+str(self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize)*self.params['bin'] proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) proccmd += " last="+str(self.params['numpart']-1) apEMAN.executeEmanCmd(proccmd, verbose=True) ### process stack to final file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") proccmd = "proc2d temp.hed "+self.params['localstack']+" apix="+str(self.stack['apix']*self.params['bin']) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) apEMAN.executeEmanCmd(proccmd, verbose=True) apFile.removeStack("temp.hed") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) ### convert stack into single spider files templateselfile = self.initializeTemplates() ### setup Xmipp command aligntime = time.time() xmippopts = ( " " +" -i "+os.path.join(self.params['rundir'], self.partlistdocfile) +" -iter "+str(self.params['maxiter']) +" -ref "+templateselfile +" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp) +" -psi_step "+str(self.params['psistep']) ) ### fast mode if self.params['fast'] is True: xmippopts += " -fast " if self.params['fastmode'] == "narrow": xmippopts += " -C 1e-10 " elif self.params['fastmode'] == "wide": xmippopts += " -C 1e-18 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " ### normalization if self.params['norm'] is True: xmippopts += " -norm " ### find number of processors if self.params['nproc'] is None: nproc = nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] mpirun = self.checkMPI() if nproc > 2 and mpirun is not None: ### use multi-processor apDisplay.printColor("Using "+str(nproc-1)+" processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True) mpiruncmd = mpirun+" -np "+str(nproc-1)+" "+xmippexe+" "+xmippopts self.writeXmippLog(mpiruncmd) apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True) else: ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe+" "+xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ### minor post-processing self.createReferenceStack() self.readyUploadFlag() self.dumpParameters()
def start(self): self.insertMaxLikeJob() self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.estimateIterTime() self.dumpParameters() ### process stack to local temp file proccmd = "proc2d " + self.stack['file'] + " temp.hed apix=" + str( self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], clipsize, clipsize) proccmd += " last=" + str(self.params['numpart'] - 1) apEMAN.executeEmanCmd(proccmd, verbose=True) ### process stack to final file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp + ".hed") proccmd = "proc2d temp.hed " + self.params[ 'localstack'] + " apix=" + str( self.stack['apix'] * self.params['bin']) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp=" + str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp=" + str(self.params['lowpass']) apEMAN.executeEmanCmd(proccmd, verbose=True) apFile.removeStack("temp.hed") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles( self.params['localstack']) ### convert stack into single spider files templateselfile = self.initializeTemplates() ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params['rundir'], self.partlistdocfile) + " -iter " + str(self.params['maxiter']) + " -ref " + templateselfile + " -o " + os.path.join(self.params['rundir'], "part" + self.timestamp) + " -psi_step " + str(self.params['psistep'])) ### fast mode if self.params['fast'] is True: xmippopts += " -fast " if self.params['fastmode'] == "narrow": xmippopts += " -C 1e-10 " elif self.params['fastmode'] == "wide": xmippopts += " -C 1e-18 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " ### normalization if self.params['norm'] is True: xmippopts += " -norm " ### find number of processors if self.params['nproc'] is None: nproc = nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] mpirun = self.checkMPI() if nproc > 2 and mpirun is not None: ### use multi-processor apDisplay.printColor("Using " + str(nproc - 1) + " processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True) mpiruncmd = mpirun + " -np " + str( nproc - 1) + " " + xmippexe + " " + xmippopts self.writeXmippLog(mpiruncmd) apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True) else: ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe + " " + xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### minor post-processing self.createReferenceStack() self.readyUploadFlag() self.dumpParameters()
def crossCorrelateAndShift(infile, reffile, alignfile, ccdocfile, numpart, dataext=".spi"): ### rewriten to do the whole thing in memory in SPIDER, it should be faster starttime = time.time() infile = spyder.fileFilter(infile) reffile = spyder.fileFilter(reffile) alignfile = spyder.fileFilter(alignfile) partimg = "_4" ccmap = "_5" windccmap = "_6" boxsize = apFile.getBoxSize(infile+dataext) if not os.path.isfile(infile+dataext): apDisplay.printError("input stack file not found: "+infile+dataext) if not os.path.isfile(reffile+dataext): apDisplay.printError("reference stack file not found: "+reffile+dataext) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) ### Allocate empty stack mySpider.toSpiderQuiet( "MS I", #command "_2@", #name "%d,%d,%d"%(boxsize), #boxsize str(numpart+1), #num part to create in memory str(numpart+1), #max particle number ) partnum = 0 while partnum < numpart: partnum+=1 mySpider.toSpiderQuiet("CP", infile+("@%05d"%(partnum)), #picture partimg, ) ### cross correlate images; reversed order to avoid -1*shift mySpider.toSpiderQuiet("CC N", reffile+("@%05d"%(partnum)), #reference partimg, #picture ccmap, #output file ) ### cannot shift more the 1/4 size of the image mySpider.toSpiderQuiet("FI x52", partimg, "12" ) mySpider.toSpiderQuiet("x54=int(x52/2)") #window size mySpider.toSpiderQuiet("x55=int(x52/4)") #window topleft mySpider.toSpiderQuiet("WI", ccmap, #input file windccmap, #output file "x54,x54", #window size "x55,x55", #window origin ) ### find the cross-correlation peak mySpider.toSpiderQuiet("x56=int(x52/4)+1") #center of window mySpider.toSpiderQuiet("PK M x11,x12,x13,x14", windccmap, #input ccmap file "x56,x56", #origin coordinates ) ### save info to doc file mySpider.toSpiderQuiet("SD %d,x13,x14"%(partnum), ccdocfile, #input ccmap file ) ### shift the images images mySpider.toSpiderQuiet("SH", partimg, #old stack ("_2@%05d"%(partnum)), #new stack "x13,x14", #shift value file ) ### finish up #save stack to file mySpider.toSpiderQuiet( "CP", "_2@", alignfile+"@", ) #delete stack mySpider.toSpiderQuiet( "DE", "_2", ) mySpider.close() apDisplay.printColor("finished shifting particles in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def start(self): self.runtime = 0 self.checkCoranRun() ### convert stack to spider self.alignstackdata = self.getAlignedStack() maskpixrad = self.params['maskrad'] / self.alignstackdata[ 'pixelsize'] / self.params['bin'] boxpixdiam = int(math.ceil(maskpixrad) + 1) * 2 if boxpixdiam * self.params['bin'] > self.alignstackdata['boxsize']: boxpixdiam = math.floor(self.alignstackdata['boxsize'] / self.params['bin']) clippixdiam = boxpixdiam * self.params['bin'] apDisplay.printMsg("Pixel mask radius=" + str(maskpixrad)) oldalignedstack = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['imagicfile']) alignedstackname = re.sub("\.", "_", self.alignstackdata['imagicfile']) + ".spi" alignedstack = os.path.join(self.params['rundir'], alignedstackname) apFile.removeFile(alignedstack) emancmd = ("proc2d %s %s spiderswap shrink=%d clip=%d,%d edgenorm" % (oldalignedstack, alignedstack, self.params['bin'], clippixdiam, clippixdiam)) if self.params['numpart'] is not None: emancmd += " last=%d" % (self.params['numpart'] - 1) numpart = self.params['numpart'] else: numpart = self.getNumAlignedParticles() apEMAN.executeEmanCmd(emancmd, verbose=True) ### get number of processors nproc = apParam.getNumProcessors() esttime = classification.estimateTime(numpart, maskpixrad) apDisplay.printColor("Running spider this can take awhile, estimated time: "+\ apDisplay.timeString(esttime),"cyan") ### do correspondence analysis corantime = time.time() self.contriblist = classification.correspondenceAnalysis( alignedstack, boxsize=boxpixdiam, maskpixrad=maskpixrad, numpart=numpart, numfactors=self.params['numfactors'], nproc=nproc) corantime = time.time() - corantime ### make dendrogram dendrotime = time.time() classification.makeDendrogram(numfactors=min( 3, self.params['numfactors']), nproc=nproc) dendrotime = time.time() - dendrotime inserttime = time.time() if self.params['commit'] is True: self.runtime = corantime self.insertCoranRun(insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apFile.removeFile(alignedstack, warn=True) apDisplay.printMsg("Correspondence Analysis time: " + apDisplay.timeString(corantime)) apDisplay.printMsg("Make Dendrogram time: " + apDisplay.timeString(dendrotime)) apDisplay.printMsg("Database Insertion time: " + apDisplay.timeString(inserttime))
def start(self): self.insertTopolRepJob() self.stack = {} self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) if self.params['virtualdata'] is not None: self.stack['file'] = self.params['virtualdata']['filename'] else: self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) self.dumpParameters() self.params['canexe'] = self.getCANPath() ### process stack to local file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") a = proc2dLib.RunProc2d() a.setValue('infile',self.stack['file']) a.setValue('outfile',self.params['localstack']) a.setValue('apix',self.stack['apix']) a.setValue('bin',self.params['bin']) a.setValue('last',self.params['numpart']-1) a.setValue('append',False) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: a.setValue('lowpass',self.params['lowpass']) if self.params['highpass'] is not None and self.params['highpass'] > 1: a.setValue('highpass',self.params['highpass']) if self.params['invert'] is True: a.setValue('invert',True) if self.params['premask'] is True and self.params['mramethod'] != 'imagic': a.setValue('mask',self.params['mask']) if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) if self.params['uploadonly'] is not True: if os.path.isfile(os.path.join(self.params['rundir'],"stack.hed")): self.params['localstack']=os.path.join(self.params['rundir'],"stack.hed") else: a.run() if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']): apDisplay.printError("Missing particles in stack") ### IMAGIC mask particles before alignment if self.params['premask'] is True and self.params['mramethod'] == 'imagic': # convert mask to fraction for imagic maskfrac = self.workingmask*2/self.workingboxsize maskstack = imagicFilters.softMask(self.params['localstack'],mask=maskfrac) shutil.move(maskstack+".hed",os.path.splitext(self.params['localstack'])[0]+".hed") shutil.move(maskstack+".img",os.path.splitext(self.params['localstack'])[0]+".img") origstack = self.params['localstack'] ### find number of processors # if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() if self.params['uploadonly'] is not True: aligntime = time.time() # run through iterations for i in range(0,self.params['iter']+1): # move back to starting directory os.chdir(self.params['rundir']) # set up next iteration directory self.params['currentiter'] = i self.params['iterdir'] = os.path.abspath("iter%02i" % i) self.params['iterdir'] = string.replace(self.params['iterdir'],"/jetstor/APPION","") if os.path.exists(self.params['iterdir']): apDisplay.printError("Error: directory '%s' exists, aborting alignment" % self.params['iterdir']) # create directory for iteration os.makedirs(self.params['iterdir']) os.chdir(self.params['iterdir']) # if at first iteration, create initial class averages if i == 0: # first rewrite localstack headers if particles not pre-masked if self.params['premask'] is False and self.params['mramethod'] == "imagic": imagicFilters.takeoverHeaders(self.params['localstack'],self.params['numpart'],self.workingboxsize) self.params['alignedstack'] = os.path.splitext(self.params['localstack'])[0] if self.params['msamethod']=='imagic': self.runIMAGICmsa() else: self.runCAN() continue # using references from last iteration, run multi-ref alignment if self.params['mramethod'] == "imagic": # rewrite class headers imagicFilters.takeoverHeaders(self.params['currentcls'],self.params['currentnumclasses'],self.workingboxsize) self.runIMAGICmra() else: self.runEMANmra() # create class averages from aligned stack if self.params['msamethod']=='imagic': self.runIMAGICmsa() else: self.runCAN() aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ## set upload information params: else: ## get last iteration alliters = glob.glob("iter*") alliters.sort() ## get iteration number from iter dir self.params['currentiter'] = int(alliters[-1][-2:]) self.params['iterdir'] = os.path.join(self.params['rundir'],alliters[-1]) self.params['currentcls'] = "classes%02i"%(self.params['currentiter']) ## go into last iteration directory os.chdir(self.params['iterdir']) self.params['alignedstack'] = os.path.abspath("mrastack") if os.path.isfile(os.path.join(self.params['rundir'],self.params['currentcls']+".hed")): p1 = os.path.join(self.params['rundir'],self.params['currentcls']) p2 = os.path.join(self.params['iterdir'],self.params['currentcls']) shutil.move(p1+".hed",p2+".hed") shutil.move(p1+".img",p2+".img") ## sort the class averages self.sortClassAverages() ### get particle information from last iteration if self.params['mramethod']=='imagic': partlist = self.readPartIMAGICFile() else: partlist = self.readPartEMANFile() if self.params['msamethod']=='imagic': partrefdict = self.imagicClassificationToDict() else: partrefdict = self.canClassificationToDict() # move back to starting directory os.chdir(self.params['rundir']) # move aligned stack to current directory for appionweb if not os.path.isfile("mrastack.hed"): shutil.move(self.params['alignedstack']+".hed","mrastack.hed") shutil.move(self.params['alignedstack']+".img","mrastack.img") # rewrite header if self.params['mramethod'] == "imagic" or self.params['msamethod'] == 'imagic': imagicFilters.takeoverHeaders("mrastack",self.params['numpart'],self.workingboxsize) # move actual averages to current directory if self.params['msamethod']=='can': if not os.path.isfile("classes_avg.hed"): self.applySort() # save actual class averages as refs in database self.params['currentcls']="classes_avg" ### create an average mrc of final references if not os.path.isfile("average.mrc"): apStack.averageStack(stack=self.params['currentcls']+".hed") self.dumpParameters() ### remove the filtered stack apFile.removeStack(origstack) ### save to database if self.params['commit'] is True: self.insertRunIntoDatabase() self.insertParticlesIntoDatabase(partlist, partrefdict)
def alignStack(oldstack, alignedstack, partlist, dataext=".spi"): """ write aligned stack -- with python loop inputs: oldstack newstack (empty) list of particle dictionaries for operations modifies: newstack output: none I tried this loop in both spider and python; python was faster?!? -neil """ if not os.path.isfile(oldstack+dataext): apDisplay.printError("Could not find original stack: "+oldstack+dataext) boxsize = apFile.getBoxSize(oldstack+dataext) apDisplay.printMsg("applying alignment parameters to stack") apFile.removeFile(alignedstack+dataext) count = 0 t0 = time.time() nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) #create stack in core numpart = len(partlist) mySpider.toSpiderQuiet( "MS I", #command "_2@", #name "%d,%d,%d"%(boxsize), #boxsize str(numpart+1), #num part to create in memory str(numpart+1), #max particle number ) for partdict in partlist: partnum = partdict['num'] #if partdict['num'] in [3,6,7]: # print partdict['num'], partdict['template'], partdict['mirror'], round(partdict['rot'],3) ### Rotate and Shift operations count += 1 #rotate/shift mySpider.toSpiderQuiet( "RT SQ", spyder.fileFilter(oldstack)+"@"+("%06d" % (partnum)), "_1", str(partdict['rot']), str(partdict['xshift'])+","+str(partdict['yshift']), ) #mirror, if necessary if 'mirror' in partdict and partdict['mirror'] is True: mySpider.toSpiderQuiet( "MR", "_1", "_2@"+("%06d" % (partnum)), "Y", ) else: mySpider.toSpiderQuiet( "CP", "_1", "_2@"+("%06d" % (partnum)), ) ### finish up #save stack to file mySpider.toSpiderQuiet( "CP", "_2@", spyder.fileFilter(alignedstack)+"@", ) #delete stack mySpider.toSpiderQuiet( "DE", "_2", ) mySpider.close() apDisplay.printMsg("Completed transforming %d particles in %s"%(count, apDisplay.timeString(time.time()-t0))) if count < 1: apDisplay.printError("Failed to transform any particles") if not os.path.isfile(alignedstack+dataext): apDisplay.printError("Failed to create stack "+alignedstack+dataext) return
def refBasedAlignParticles(stackfile, templatestack, origstackfile, xysearch, xystep, numpart, numtemplate, firstring=2, lastring=100, dataext=".spi", iternum=1, oldpartlist=None): """ inputs: stack template search params outputs: aligned stack rotation/shift params """ ### setup if dataext in templatestack: templatestack = templatestack[:-4] if dataext in stackfile: stackfile = stackfile[:-4] if dataext in origstackfile: origstackfile = origstackfile[:-4] t0 = time.time() rundir = "alignments" apParam.createDirectory(rundir) nproc = apParam.getNumProcessors() ### remove previous iterations apFile.removeFile(rundir + "/paramdoc%02d%s" % (iternum, dataext)) ### perform alignment, should I use 'AP SH' instead? mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider( "AP MQ", spyder.fileFilter(templatestack) + "@**", # reference image series "1-" + str(numtemplate), # enter number of templates of doc file str(int(xysearch)) + "," + str(int(xystep)), # translation search range, step size str(int(firstring)) + "," + str(int(lastring)), # first and last ring for rotational correlation spyder.fileFilter(stackfile) + "@******", # unaligned image series "1-" + str(numpart), # enter number of particles of doc file rundir + ("/paramdoc%02d" % (iternum)), # output angles document file ) mySpider.close() """ ### perform alignment, should I use 'AP SH' instead? mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("AP SH", spyder.fileFilter(templatestack)+"@**", # reference image series "1-"+str(numtemplate), # enter number of templates of doc file str(int(xysearch))+","+str(int(xystep)), # translation search range, step size str(int(firstring))+","+str(int(lastring)), # first and last ring for rotational correlation "*", # initial angles spyder.fileFilter(stackfile)+"@******", # unaligned image series "1-"+str(numpart), # enter number of particles of doc file "*", # initial angles "0.0", # angle search "1", # check mirrors and shift/rotate input rundir+("/paramdoc%02d" % (iternum)), # output angles document file ) mySpider.close() """ ### convert spider rotation, shift data to python docfile = rundir + ("/paramdoc%02d" % (iternum)) + dataext picklefile = rundir + ("/paramdoc%02d" % (iternum)) + ".pickle" if oldpartlist is not None and iternum > 1: apDisplay.printMsg("updating particle doc info") partlist = updateRefBasedDocFile(oldpartlist, docfile, picklefile) elif iternum == 1: apDisplay.printMsg("reading initial particle doc info") partlist = readRefBasedDocFile(docfile, picklefile) else: apDisplay.printError( "reading (not updating) particle doc info on iteration " + str(iternum)) ### write aligned stack -- with python loop alignedstack = rundir + ("/alignedstack%02d" % (iternum)) alignStack(origstackfile, alignedstack, partlist, dataext) ### average stack emancmd = ("proc2d " + alignedstack + dataext + " " + rundir + ("/avgimg%02d" % (iternum)) + ".mrc " + " average") apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) td1 = time.time() - t0 apDisplay.printMsg("completed alignment of " + str(numpart) + " particles in " + apDisplay.timeString(td1)) if numpart < 1: apDisplay.printError("Failed to find any particles") return alignedstack + dataext, partlist
def checkConflicts(self): ### first get all stack data self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### modify boxsize and pixelsize according to binning self.boxsize = self.stack['boxsize'] self.clipsize = int( math.floor(self.boxsize / float(self.params['bin'] * 2))) * 2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" % (self.params['clipsize'], self.clipsize)) self.clipsize = self.params['clipsize'] self.apix = self.stack['apix'] if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even self.boxsize = clipsize self.apix = self.apix * self.params['bin'] ### basic error checking if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['ncls'] is None: apDisplay.printError("a number of classes was not provided") maxparticles = 1000000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile))) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) if self.params['numpart'] > 5000: apDisplay.printWarning( "initial model calculation may not work with less than 5000 particles" ) self.mpirun = self.checkMPI() # if self.mpirun is None: # apDisplay.printError("There is no MPI installed") if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() ### SIMPLE defaults and error checking if self.params['ring2'] is None: self.params['ring2'] = (self.boxsize / 2) - 2 if self.params['ncls'] > 2000: apDisplay.printError( "number of classes should be less than 2000 for subsequent ORIGAMI run to work" ) if self.params['ncls'] > self.params['numpart']: self.params['ncls'] = self.params['numpart'] / self.params['minp'] if self.params['mask'] is None: self.params['mask'] = (self.boxsize / 2) - 2 if self.params['mw'] is None: apDisplay.printError( "please specify the molecular weight (in kDa)")
def refBasedAlignParticles(stackfile, templatestack, origstackfile, xysearch, xystep, numpart, numtemplate, firstring=2, lastring=100, dataext=".spi", iternum=1, oldpartlist=None): """ inputs: stack template search params outputs: aligned stack rotation/shift params """ ### setup if dataext in templatestack: templatestack = templatestack[:-4] if dataext in stackfile: stackfile = stackfile[:-4] if dataext in origstackfile: origstackfile = origstackfile[:-4] t0 = time.time() rundir = "alignments" apParam.createDirectory(rundir) nproc = apParam.getNumProcessors() ### remove previous iterations apFile.removeFile(rundir+"/paramdoc%02d%s" % (iternum, dataext)) ### perform alignment, should I use 'AP SH' instead? mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("AP MQ", spyder.fileFilter(templatestack)+"@**", # reference image series "1-"+str(numtemplate), # enter number of templates of doc file str(int(xysearch))+","+str(int(xystep)), # translation search range, step size str(int(firstring))+","+str(int(lastring)), # first and last ring for rotational correlation spyder.fileFilter(stackfile)+"@******", # unaligned image series "1-"+str(numpart), # enter number of particles of doc file rundir+("/paramdoc%02d" % (iternum)), # output angles document file ) mySpider.close() """ ### perform alignment, should I use 'AP SH' instead? mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("AP SH", spyder.fileFilter(templatestack)+"@**", # reference image series "1-"+str(numtemplate), # enter number of templates of doc file str(int(xysearch))+","+str(int(xystep)), # translation search range, step size str(int(firstring))+","+str(int(lastring)), # first and last ring for rotational correlation "*", # initial angles spyder.fileFilter(stackfile)+"@******", # unaligned image series "1-"+str(numpart), # enter number of particles of doc file "*", # initial angles "0.0", # angle search "1", # check mirrors and shift/rotate input rundir+("/paramdoc%02d" % (iternum)), # output angles document file ) mySpider.close() """ ### convert spider rotation, shift data to python docfile = rundir+("/paramdoc%02d" % (iternum))+dataext picklefile = rundir+("/paramdoc%02d" % (iternum))+".pickle" if oldpartlist is not None and iternum > 1: apDisplay.printMsg("updating particle doc info") partlist = updateRefBasedDocFile(oldpartlist, docfile, picklefile) elif iternum == 1: apDisplay.printMsg("reading initial particle doc info") partlist = readRefBasedDocFile(docfile, picklefile) else: apDisplay.printError("reading (not updating) particle doc info on iteration "+str(iternum)) ### write aligned stack -- with python loop alignedstack = rundir+("/alignedstack%02d" % (iternum)) alignStack(origstackfile, alignedstack, partlist, dataext) ### average stack emancmd = ( "proc2d "+alignedstack+dataext+" " +rundir+("/avgimg%02d" % (iternum))+".mrc " +" average") apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) td1 = time.time()-t0 apDisplay.printMsg("completed alignment of "+str(numpart) +" particles in "+apDisplay.timeString(td1)) if numpart < 1: apDisplay.printError("Failed to find any particles") return alignedstack+dataext, partlist