def __init__(self, optargs=sys.argv[1:], quiet=False, useglobalparams=True, maxnproc=None): """ Starts a new function and gets all the parameters """ ### setup some expected values self.successful_run = False self.params = {} sys.stdout.write("\n\n") self.quiet = quiet self.maxnproc = maxnproc self.startmem = mem.active() self.t0 = time.time() self.createDefaultStats() self.timestamp = apParam.makeTimestamp() self.argdict = {} self.optdict = {} apDisplay.printMsg("Time stamp: " + self.timestamp) self.functionname = apParam.getFunctionName(sys.argv[0]) apDisplay.printMsg("Function name: " + self.functionname) self.appiondir = apParam.getAppionDirectory() apDisplay.printMsg("Appion directory: " + self.appiondir) self.parsePythonPath() # loadavg = os.getloadavg()[0] # if loadavg > 2.0: # apDisplay.printMsg("Load average is high "+str(round(loadavg,2))) # loadsquared = loadavg*loadavg # time.sleep(loadavg) # apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2))) self.setLockname('lock') ### setup default parser: run directory, etc. self.setParams(optargs, useglobalparams) #if 'outdir' in self.params and self.params['outdir'] is not None: # self.params['rundir'] = self.params['outdir'] self.checkConflicts() if useglobalparams is True: self.checkGlobalConflicts() ### setup run directory self.setProcessingDirName() self.setupRunDirectory() ### Start pool of threads to run subprocesses. ### Later you will use self.process_launcher.launch(...) to ### put commands into the queue. ### There is currently a timeout built into it that will cause ### the threads to die if they have no tasks after 10 seconds. self.process_launcher = apThread.ProcessLauncher( 2, self.params['rundir']) ### write function log self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet)) ### any custom init functions go here self.onInit()
def __init__(self,optargs=sys.argv[1:],quiet=False): """ Starts a new function and gets all the parameters """ ### setup some expected values self.startmem = mem.active() self.t0 = time.time() self.createDefaultStats() self.quiet = quiet self.timestamp = apParam.makeTimestamp() if not self.quiet: apDisplay.printMsg("Time stamp: "+self.timestamp) self.functionname = apParam.getFunctionName(sys.argv[0]) if not self.quiet: apDisplay.printMsg("Function name: "+self.functionname) apParam.setUmask() self.parsePythonPath() loadavg = os.getloadavg()[0] if loadavg > 2.0: apDisplay.printMsg("Load average is %.2f, wait for %.1f second " % (round(loadavg,2),loadavg**2)) time.sleep(loadavg**2) apDisplay.printMsg("Load average is high "+str(round(loadavg,2))) ### setup default parser: run directory, etc. self.setParams(optargs) self.checkConflicts() ### write function log self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet)) ### any custom init functions go here self.onInit()
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"): """ inputs: aligned particle stack number of classes timestamp output directory output: class averages class variances dendrogram.png """ if timestamp is None: timestamp = apParam.makeTimestamp() classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses)) classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses)) thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext) ### create class doc files mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True) mySpider.toSpider( "CL HE", thresh, spyder.fileFilter(dendrogramfile), # dendrogram doc file rundir+"/classdoc_"+timestamp+"_****", # class doc file ) ### delete existing files sys.stderr.write("delete existing files") for dext in (".hed", ".img", dataext): apFile.removeFile(classavg+dext) apFile.removeFile(classvar+dext) print "" ### create class averages sys.stderr.write("create class averages") for i in range(classes): sys.stderr.write(".") classnum = i+1 mySpider.toSpiderQuiet( "AS R", spyder.fileFilter(alignedstack)+"@******", rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)), "A", (classavg+"@%04d" % (classnum)), (classvar+"@%04d" % (classnum)), ) mySpider.close() print "" ### convert to IMAGIC emancmd = "proc2d "+classavg+".spi "+classavg+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) emancmd = "proc2d "+classvar+".spi "+classvar+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return classavg,classvar
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"): """ inputs: aligned particle stack number of classes timestamp output directory output: class averages class variances dendrogram.png """ if timestamp is None: timestamp = apParam.makeTimestamp() classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses)) classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses)) thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext) ### create class doc files mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True) mySpider.toSpider( "CL HE", thresh, spyder.fileFilter(dendrogramfile), # dendrogram doc file rundir+"/classdoc_"+timestamp+"_****", # class doc file ) ### delete existing files sys.stderr.write("delete existing files") for dext in (".hed", ".img", dataext): apFile.removeFile(classavg+dext) apFile.removeFile(classvar+dext) print "" ### create class averages sys.stderr.write("create %i class averages"%classes) for i in range(classes): sys.stderr.write(".") classnum = i+1 mySpider.toSpiderQuiet( "AS R", spyder.fileFilter(alignedstack)+"@******", rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)), "A", (classavg+"@%04d" % (classnum)), (classvar+"@%04d" % (classnum)), ) mySpider.close() print "" ### convert to IMAGIC emancmd = "proc2d "+classavg+".spi "+classavg+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) emancmd = "proc2d "+classvar+".spi "+classvar+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return classavg,classvar
def __init__(self,optargs=sys.argv[1:],quiet=False,useglobalparams=True,maxnproc=None): """ Starts a new function and gets all the parameters """ ### setup some expected values self.successful_run = False self.params = {} sys.stdout.write("\n\n") self.quiet = quiet self.maxnproc = maxnproc self.startmem = mem.active() self.t0 = time.time() self.createDefaultStats() self.timestamp = apParam.makeTimestamp() self.argdict = {} self.optdict = {} apDisplay.printMsg("Time stamp: "+self.timestamp) self.functionname = apParam.getFunctionName(sys.argv[0]) apDisplay.printMsg("Function name: "+self.functionname) self.appiondir = apParam.getAppionDirectory() apDisplay.printMsg("Appion directory: "+self.appiondir) self.parsePythonPath() # loadavg = os.getloadavg()[0] # if loadavg > 2.0: # apDisplay.printMsg("Load average is high "+str(round(loadavg,2))) # loadsquared = loadavg*loadavg # time.sleep(loadavg) # apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2))) self.setLockname('lock') ### setup default parser: run directory, etc. self.setParams(optargs,useglobalparams) #if 'outdir' in self.params and self.params['outdir'] is not None: # self.params['rundir'] = self.params['outdir'] self.checkConflicts() if useglobalparams is True: self.checkGlobalConflicts() ### setup run directory self.setProcessingDirName() self.setupRunDirectory() ### Start pool of threads to run subprocesses. ### Later you will use self.process_launcher.launch(...) to ### put commands into the queue. ### There is currently a timeout built into it that will cause ### the threads to die if they have no tasks after 10 seconds. self.process_launcher = apThread.ProcessLauncher(2, self.params['rundir']) ### write function log self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet)) ### any custom init functions go here self.onInit()
def copyTemplatesToOutdir(params, timestamp=None): newlist = [] for tmpl in params['templatelist']: base = os.path.basename(tmpl) old = os.path.abspath(tmpl) ### Rename file for new location if timestamp is None: timestamp = apParam.makeTimestamp() #append the name of the directory to the filename #basedir = os.path.split(os.path.dirname(old))[1] #base = basedir+"_"+base name, ext = os.path.splitext(base) base = name + "-" + timestamp + ext new = os.path.join(params['rundir'], base) if os.path.isfile(new): mdnew = apFile.md5sumfile(new) mdold = apFile.md5sumfile(old) if mdnew != mdold: apDisplay.printError("a different template with name \'" + new + "\' already exists!") elif apDatabase.isTemplateInDB(mdnew): apDisplay.printWarning("same template with md5sum \'" + mdnew + "\' already exists in the DB!") apDisplay.printMsg("skipping template \'" + old + "\'") else: apDisplay.printWarning("the same template with name \'" + new + "\' already exists!") newlist.append(base) else: #template is okay to copy and insert apDisplay.printMsg("copying file " + old + " to " + new) shutil.copyfile(old, new) newlist.append(base) #and only allow user read access just so they don't get deleted #os.lchmod(new, 0666) os.chmod(new, 0666) params['templatelist'] = newlist apDisplay.printColor("New template List:", "green") pprint.pprint(params['templatelist']) return
def copyTemplatesToOutdir(params, timestamp=None): newlist = [] for tmpl in params['templatelist']: base = os.path.basename(tmpl) old = os.path.abspath(tmpl) ### Rename file for new location if timestamp is None: timestamp = apParam.makeTimestamp() #append the name of the directory to the filename #basedir = os.path.split(os.path.dirname(old))[1] #base = basedir+"_"+base name,ext = os.path.splitext(base) base = name+"-"+timestamp+ext new = os.path.join(params['rundir'], base) if os.path.isfile(new): mdnew = apFile.md5sumfile(new) mdold = apFile.md5sumfile(old) if mdnew != mdold: apDisplay.printError("a different template with name \'"+new+"\' already exists!") elif apDatabase.isTemplateInDB(mdnew): apDisplay.printWarning("same template with md5sum \'"+mdnew+"\' already exists in the DB!") apDisplay.printMsg("skipping template \'"+old+"\'") else: apDisplay.printWarning("the same template with name \'"+new+"\' already exists!") newlist.append(base) else: #template is okay to copy and insert apDisplay.printMsg("copying file "+old+" to "+new) shutil.copyfile(old, new) newlist.append(base) #and only allow user read access just so they don't get deleted #os.lchmod(new, 0666) os.chmod(new, 0666) params['templatelist'] = newlist apDisplay.printColor("New template List:","green") pprint.pprint(params['templatelist']) return
def checkConflicts(self): ### unpickle results or parse logfile, set default parameters if missing os.chdir(os.path.abspath(self.params['rundir'])) if os.path.isdir(os.path.join(self.params['rundir'], "recon")): apDisplay.printWarning( 'recon dir already exist, no need to unpack') else: self.unpackResults() ''' These are required error checks, everything else should be possible to obtain from the timestamped pickle file ''' stackdata = apStack.getOnlyStackData(self.params['stackid']) self.runparams = {} self.runparams['original_apix'] = ( stackdata['pixelsize'] / 1e-10 ) # convert pixelsize to angstroms per pixel. self.runparams['original_boxsize'] = stackdata['boxsize'] # some functions in readRunParameters needs the above params self.runparams.update(self.readRunParameters()) ### parameters recovered from runparameter file(s) if not self.runparams.has_key('stackid'): if self.params['stackid'] is not None: self.runparams['stackid'] = self.params['stackid'] else: apDisplay.printError( "stack id must be specified for proper database insertion") if not self.runparams.has_key('modelid'): if self.params['modelid'] is not None: self.runparams['modelid'] = self.params['modelid'] else: apDisplay.printError( "model id must be specified for proper database insertion") if not self.runparams.has_key('NumberOfReferences'): if self.params['NumberOfReferences'] is not None: self.runparams['NumberOfReferences'] = self.params[ 'NumberOfReferences'] else: apDisplay.printError( "number of references produced during the refinement needs to be specified" ) if not self.runparams.has_key('numiter'): if self.params['numiter'] is not None: self.runparams['numiter'] = self.params['numiter'] else: apDisplay.printError( "number of iterations run during the refinement needs to be specified" ) if not self.runparams.has_key('boxsize'): if self.params['boxsize'] is not None: self.runparams['boxsize'] = self.params['boxsize'] else: apDisplay.printError( "boxsize of the map / particles submitted for refinement needs to be specified" ) if not self.runparams.has_key('apix'): if self.params['apix'] is not None: self.runparams['apix'] = self.params['apix'] else: apDisplay.printError( "pixelsize of the map / particles submitted for refinement needs to be specified" ) if not self.runparams.has_key('symmetry'): if self.params['symid'] is not None: self.runparams['symmetry'] = apSymmetry.getSymmetryDataFromID( self.params['symid']) else: apDisplay.printError( "symmetry ID must be specified, you can input --symid=25 for an asymmetric reconstruction" ) # access multiModelRefinementRun this way in case it is not present if 'multiModelRefinementRun' in vars(self): if not self.runparams.has_key( 'NumberOfReferences' ) and self.multiModelRefinementRun is True: if self.params['NumberOfReferences'] is not None: self.runparams['NumberOfReferences'] = self.params[ 'NumberOfReferences'] else: apDisplay.printError( "number of output models in refinement needs to be specified for multi-model run" ) else: if self.params['NumberOfReferences'] is not None: self.runparams['NumberOfReferences'] = self.params[ 'NumberOfReferences'] if self.runparams['NumberOfReferences'] > 1: self.multiModelRefinementRun = True else: self.multiModelRefinementRun = False else: apDisplay.printError( "number of output models (references) in refinement needs to be specified for multi-model run" ) if not self.runparams.has_key('rundir'): self.runparams['rundir'] = self.params['rundir'] if not self.runparams.has_key('reconstruction_working_dir'): self.runparams['reconstruction_working_dir'] = str( self.package) + "_results" if not self.runparams.has_key('mask'): self.runparams['mask'] = None if not self.runparams.has_key('imask'): self.runparams['imask'] = None if not self.runparams.has_key('alignmentInnerRadius'): self.runparams['alignmentInnerRadius'] = None if not self.runparams.has_key('alignmentOuterRadius'): self.runparams['alignmentOuterRadius'] = None if not self.runparams.has_key('angularSamplingRate'): self.runparams['angularSamplingRate'] = None ### parameters specified for upload if self.params['jobid'] is not None: self.params['jobinfo'] = appiondata.ApAppionJobData.direct_query( self.params['jobid']) else: jobid = self.tryToGetJobID() if jobid is not False: self.params[ 'jobinfo'] = appiondata.ApAppionJobData.direct_query( self.params['jobid']) else: self.params['jobinfo'] = None if self.params[ 'timestamp'] is None and self.package == "external_package": apDisplay.printError("a timestamp (or some identifier) must be specified with the files. For example, the 3D mrc file " \ "for iteration 1 should be named 'recon_YourIdentifier_it001_vol001.mrc, in which case the timestamp should be specified " \ "as --timestamp=YourIdentifier") elif self.params['timestamp'] is None and self.params[ 'jobid'] is None and self.package != "external_package": self.params['timestamp'] = apParam.makeTimestamp() elif self.params['timestamp'] is None and self.params[ 'jobid'] is not None: timestamp = self.getTimestamp() if timestamp is None: self.params['timestamp'] = apParam.makeTimestamp() # apDisplay.printError("please specify the timestamp associated with the refinement parameters, e.g. --timestamp=08nov25c07") ### basic refinement variables self.initializeRefinementUploadVariables() return
def kmeansCluster(alignedstack, numpart=None, numclasses=40, timestamp=None, factorlist=range(1,5), corandata="coran/corandata", dataext=".spi"): """ inputs: outputs: """ if timestamp is None: timestamp = apParam.makeTimestamp() if alignedstack[-4:] == dataext: alignedstack = alignedstack[:-4] rundir = "cluster" classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses)) classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses)) apParam.createDirectory(rundir) for i in range(numclasses): apFile.removeFile(rundir+("/classdoc%04d" % (i+1))+dataext) apFile.removeFile(rundir+("/allclassesdoc%04d" % (numclasses))+dataext) ### make list of factors factorstr, factorkey = operations.intListToString(factorlist) ### do k-means clustering mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) mySpider.toSpider( "CL KM", corandata+"_IMC", # path to coran data str(numclasses), # num classes factorstr, # factor string ) ## weight for each factor for fact in factorlist: mySpider.toSpiderQuiet("1.0") randnum = (int(random.random()*1000) + 1) mySpider.toSpider( str(randnum), rundir+"/classdoc_"+timestamp+"_****", # class doc file rundir+("/allclassesdoc%04d" % (numclasses)), #clusterdoc file ) mySpider.close() ### delete existing files sys.stderr.write("delete existing files") for dext in (".hed", ".img", dataext): apFile.removeFile(classavg+dext) apFile.removeFile(classvar+dext) print "" mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) ### create class averages apDisplay.printMsg("Averaging particles into classes") for i in range(numclasses): classnum = i+1 mySpider.toSpiderQuiet( "AS R", spyder.fileFilter(alignedstack)+"@******", rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)), "A", (classavg+"@%04d" % (classnum)), (classvar+"@%04d" % (classnum)), ) if classnum % 10 == 0: sys.stderr.write(".") time.sleep(1) mySpider.close() ### convert to IMAGIC emancmd = "proc2d "+classavg+".spi "+classavg+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) emancmd = "proc2d "+classvar+".spi "+classvar+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return classavg,classvar
def checkConflicts(self): ### unpickle results or parse logfile, set default parameters if missing os.chdir(os.path.abspath(self.params["rundir"])) if os.path.isdir(os.path.join(self.params["rundir"], "recon")): apDisplay.printWarning("recon dir already exist, no need to unpack") else: self.unpackResults() """ These are required error checks, everything else should be possible to obtain from the timestamped pickle file """ stackdata = apStack.getOnlyStackData(self.params["stackid"]) self.runparams = {} self.runparams["original_apix"] = stackdata["pixelsize"] / 1e-10 # convert pixelsize to angstroms per pixel. self.runparams["original_boxsize"] = stackdata["boxsize"] # some functions in readRunParameters needs the above params self.runparams.update(self.readRunParameters()) ### parameters recovered from runparameter file(s) if not self.runparams.has_key("stackid"): if self.params["stackid"] is not None: self.runparams["stackid"] = self.params["stackid"] else: apDisplay.printError("stack id must be specified for proper database insertion") if not self.runparams.has_key("modelid"): if self.params["modelid"] is not None: self.runparams["modelid"] = self.params["modelid"] else: apDisplay.printError("model id must be specified for proper database insertion") if not self.runparams.has_key("NumberOfReferences"): if self.params["NumberOfReferences"] is not None: self.runparams["NumberOfReferences"] = self.params["NumberOfReferences"] else: apDisplay.printError("number of references produced during the refinement needs to be specified") if not self.runparams.has_key("numiter"): if self.params["numiter"] is not None: self.runparams["numiter"] = self.params["numiter"] else: apDisplay.printError("number of iterations run during the refinement needs to be specified") if not self.runparams.has_key("boxsize"): if self.params["boxsize"] is not None: self.runparams["boxsize"] = self.params["boxsize"] else: apDisplay.printError("boxsize of the map / particles submitted for refinement needs to be specified") if not self.runparams.has_key("apix"): if self.params["apix"] is not None: self.runparams["apix"] = self.params["apix"] else: apDisplay.printError("pixelsize of the map / particles submitted for refinement needs to be specified") if not self.runparams.has_key("symmetry"): if self.params["symid"] is not None: self.runparams["symmetry"] = apSymmetry.getSymmetryDataFromID(self.params["symid"]) else: apDisplay.printError( "symmetry ID must be specified, you can input --symid=25 for an asymmetric reconstruction" ) # access multiModelRefinementRun this way in case it is not present if "multiModelRefinementRun" in vars(self): if not self.runparams.has_key("NumberOfReferences") and self.multiModelRefinementRun is True: if self.params["NumberOfReferences"] is not None: self.runparams["NumberOfReferences"] = self.params["NumberOfReferences"] else: apDisplay.printError( "number of output models in refinement needs to be specified for multi-model run" ) else: if self.params["NumberOfReferences"] is not None: self.runparams["NumberOfReferences"] = self.params["NumberOfReferences"] if self.runparams["NumberOfReferences"] > 1: self.multiModelRefinementRun = True else: self.multiModelRefinementRun = False else: apDisplay.printError( "number of output models (references) in refinement needs to be specified for multi-model run" ) if not self.runparams.has_key("rundir"): self.runparams["rundir"] = self.params["rundir"] if not self.runparams.has_key("reconstruction_working_dir"): self.runparams["reconstruction_working_dir"] = str(self.package) + "_results" if not self.runparams.has_key("mask"): self.runparams["mask"] = None if not self.runparams.has_key("imask"): self.runparams["imask"] = None if not self.runparams.has_key("alignmentInnerRadius"): self.runparams["alignmentInnerRadius"] = None if not self.runparams.has_key("alignmentOuterRadius"): self.runparams["alignmentOuterRadius"] = None if not self.runparams.has_key("angularSamplingRate"): self.runparams["angularSamplingRate"] = None ### parameters specified for upload if self.params["jobid"] is not None: self.params["jobinfo"] = appiondata.ApAppionJobData.direct_query(self.params["jobid"]) else: jobid = self.tryToGetJobID() if jobid is not False: self.params["jobinfo"] = appiondata.ApAppionJobData.direct_query(self.params["jobid"]) else: self.params["jobinfo"] = None if self.params["timestamp"] is None and self.package == "external_package": apDisplay.printError( "a timestamp (or some identifier) must be specified with the files. For example, the 3D mrc file " "for iteration 1 should be named 'recon_YourIdentifier_it001_vol001.mrc, in which case the timestamp should be specified " "as --timestamp=YourIdentifier" ) elif self.params["timestamp"] is None and self.params["jobid"] is None and self.package != "external_package": self.params["timestamp"] = apParam.makeTimestamp() elif self.params["timestamp"] is None and self.params["jobid"] is not None: timestamp = self.getTimestamp() if timestamp is None: self.params["timestamp"] = apParam.makeTimestamp() # apDisplay.printError("please specify the timestamp associated with the refinement parameters, e.g. --timestamp=08nov25c07") ### basic refinement variables self.initializeRefinementUploadVariables() return
def createProjections(self, pad=False, invert=False): timestamp = apParam.makeTimestamp() eulerlist = self.setEulers() eulerfile = os.path.join(self.params['rundir'], "eulers.lst") f = open(eulerfile, "w") projcount = numpy.zeros((len(eulerlist)), dtype=numpy.uint16) angsum = numpy.zeros((len(eulerlist),3), dtype=numpy.float32) t0 = time.time() for i in range(self.params['projcount']): projnum = int(random.random()*len(eulerlist)) alt = random.gauss(eulerlist[projnum][0], self.params['projstdev']) az = random.gauss(eulerlist[projnum][1], self.params['projstdev']) #phi = random.random()*360.0-180.0 #phi = random.random()*360.0 if self.params['rotang'] != 0: phi = random.uniform(-1*self.params['rotang'], self.params['rotang']) else: phi = 0.0 f.write("%.8f\t%.8f\t%.8f\n"%(alt,az,phi)) ### stats projcount[projnum] += 1 angsum[projnum,0] += alt angsum[projnum,1] += az angsum[projnum,2] += phi apDisplay.printMsg("Finished random in %s, %.3f ns per iteration" %(apDisplay.timeString(time.time()-t0), 1.0e6 * (time.time()-t0)/float(self.params['projcount']))) f.close() print "projection count", projcount for i in range(len(eulerlist)): angavg = angsum[i,:]/projcount[i] print "angle average %d: %03.3f, %03.3f, %03.3f"%(i, angavg[0], angavg[1], angavg[2]) ### first get rid of projection artifacts from insufficient padding if self.params['threedfile'] is not None: origfile = self.params['threedfile'] elif self.params['modelid'] is not None: self.modelparams = appiondata.ApInitialModelData.direct_query(self.params['modelid']) origfile = os.path.join(self.modelparams['path']['path'], self.modelparams['name']) if self.params['apix'] is None: self.params['apix'] = self.modelparams['pixelsize'] if self.params['box'] is None: self.params['box'] = self.modelparams['boxsize'] clipped = os.path.join(self.params['rundir'], "clipped.mrc") newsize = self.params['box'] * 1.5 emancmd = "proc3d "+origfile+" "+clipped+" clip="+str(int(newsize))+","+str(int(newsize))+","+str(int(newsize))+" edgenorm" apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True) ### project resized file filename = os.path.join(self.params['rundir'], 'proj.img') apFile.removeStack(filename) emancmd = "project3d "+clipped+" out="+filename+" list="+eulerfile t0 = time.time() apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True) apDisplay.printMsg("Finished project3d in %s, %.3f ms per iteration" %(apDisplay.timeString(time.time()-t0), 1.0e3 * (time.time()-t0)/float(self.params['projcount']))) ### post-process stack self.post_process_stack(filename, pad, invert) return filename
def kmeansCluster(alignedstack, numpart=None, numclasses=40, timestamp=None, factorlist=range(1, 5), corandata="coran/corandata", dataext=".spi"): """ inputs: outputs: """ if timestamp is None: timestamp = apParam.makeTimestamp() if alignedstack[-4:] == dataext: alignedstack = alignedstack[:-4] rundir = "cluster" classavg = rundir + "/" + ("classavgstack_%s_%03d" % (timestamp, numclasses)) classvar = rundir + "/" + ("classvarstack_%s_%03d" % (timestamp, numclasses)) apParam.createDirectory(rundir) for i in range(numclasses): apFile.removeFile(rundir + ("/classdoc%04d" % (i + 1)) + dataext) apFile.removeFile(rundir + ("/allclassesdoc%04d" % (numclasses)) + dataext) ### make list of factors factorstr, factorkey = operations.intListToString(factorlist) ### do k-means clustering mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) mySpider.toSpider( "CL KM", corandata + "_IMC", # path to coran data str(numclasses), # num classes factorstr, # factor string ) ## weight for each factor for fact in factorlist: mySpider.toSpiderQuiet("1.0") randnum = (int(random.random() * 1000) + 1) mySpider.toSpider( str(randnum), rundir + "/classdoc_" + timestamp + "_****", # class doc file rundir + ("/allclassesdoc%04d" % (numclasses)), #clusterdoc file ) mySpider.close() ### delete existing files sys.stderr.write("delete existing files") for dext in (".hed", ".img", dataext): apFile.removeFile(classavg + dext) apFile.removeFile(classvar + dext) print "" mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) ### create class averages apDisplay.printMsg("Averaging particles into classes") for i in range(numclasses): classnum = i + 1 mySpider.toSpiderQuiet( "AS R", spyder.fileFilter(alignedstack) + "@******", rundir + ("/classdoc_" + timestamp + "_%04d" % (classnum)), "A", (classavg + "@%04d" % (classnum)), (classvar + "@%04d" % (classnum)), ) if classnum % 10 == 0: sys.stderr.write(".") time.sleep(1) mySpider.close() ### convert to IMAGIC emancmd = "proc2d " + classavg + ".spi " + classavg + ".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) emancmd = "proc2d " + classvar + ".spi " + classvar + ".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return classavg, classvar
def __init__(self, optargs=sys.argv[1:], quiet=False, useglobalparams=True, maxnproc=None): """ Starts a new function and gets all the parameters """ ### setup some expected values self.successful_run = False self.clusterjobdata = None self.params = {} sys.stdout.write("\n\n") self.quiet = quiet self.maxnproc = maxnproc self.startmem = mem.active() self.t0 = time.time() self.createDefaultStats() self.timestamp = apParam.makeTimestamp() self.argdict = {} self.optdict = {} apDisplay.printMsg("Time stamp: " + self.timestamp) self.functionname = apParam.getFunctionName(sys.argv[0]) apDisplay.printMsg("Function name: " + self.functionname) self.appiondir = apParam.getAppionDirectory() apDisplay.printMsg("Appion directory: " + self.appiondir) self.parsePythonPath() loadavg = os.getloadavg()[0] if loadavg > 2.0: apDisplay.printMsg("Load average is high " + str(round(loadavg, 2))) loadsquared = loadavg * loadavg time.sleep(loadavg) apDisplay.printMsg("New load average " + str(round(os.getloadavg()[0], 2))) self.setLockname('lock') ### setup default parser: run directory, etc. self.setParams(optargs, useglobalparams) #if 'outdir' in self.params and self.params['outdir'] is not None: # self.params['rundir'] = self.params['outdir'] ### setup correct database after we have read the project id if 'projectid' in self.params and self.params['projectid'] is not None: apDisplay.printMsg("Using split database") # use a project database newdbname = apProject.getAppionDBFromProjectId( self.params['projectid']) sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '" + newdbname + "'", "green") ### check if user wants to print help message if 'commit' in self.params and self.params['commit'] is True: apDisplay.printMsg("Committing data to database") else: apDisplay.printWarning("Not committing data to database") self.checkConflicts() if useglobalparams is True: self.checkGlobalConflicts() ### setup run directory self.setProcessingDirName() self.setupRunDirectory() ### Start pool of threads to run subprocesses. ### Later you will use self.process_launcher.launch(...) to ### put commands into the queue. ### There is currently a timeout built into it that will cause ### the threads to die if they have no tasks after 10 seconds. self.process_launcher = apThread.ProcessLauncher( 2, self.params['rundir']) ### write function log self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet)) ### upload command line parameters to database self.uploadScriptData() ### any custom init functions go here self.onInit()
def __init__(self,optargs=sys.argv[1:],quiet=False,useglobalparams=True,maxnproc=None): """ Starts a new function and gets all the parameters """ ### setup some expected values self.successful_run = False self.clusterjobdata = None self.params = {} sys.stdout.write("\n\n") self.quiet = quiet self.maxnproc = maxnproc self.startmem = mem.active() self.t0 = time.time() self.createDefaultStats() self.timestamp = apParam.makeTimestamp() self.argdict = {} self.optdict = {} apDisplay.printMsg("Time stamp: "+self.timestamp) self.functionname = apParam.getFunctionName(sys.argv[0]) apDisplay.printMsg("Function name: "+self.functionname) self.appiondir = apParam.getAppionDirectory() apDisplay.printMsg("Appion directory: "+self.appiondir) hostname = apParam.getHostname() apDisplay.printMsg("Processing hostname: "+hostname) self.parsePythonPath() # loadavg = os.getloadavg()[0] # if loadavg > 2.0: # apDisplay.printMsg("Load average is high "+str(round(loadavg,2))) # loadsquared = loadavg*loadavg # time.sleep(loadavg) # apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2))) self.setLockname('lock') ### setup default parser: run directory, etc. self.setParams(optargs,useglobalparams) #if 'outdir' in self.params and self.params['outdir'] is not None: # self.params['rundir'] = self.params['outdir'] ### setup correct database after we have read the project id if 'projectid' in self.params and self.params['projectid'] is not None: apDisplay.printMsg("Using split database") # use a project database newdbname = apProject.getAppionDBFromProjectId(self.params['projectid']) sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '"+newdbname+"'", "green") ### check if user wants to print help message if 'commit' in self.params and self.params['commit'] is True: apDisplay.printMsg("Committing data to database") else: apDisplay.printWarning("Not committing data to database") self.checkConflicts() if useglobalparams is True: self.checkGlobalConflicts() ### setup run directory self.setProcessingDirName() self.setupRunDirectory() ### Start pool of threads to run subprocesses. ### Later you will use self.process_launcher.launch(...) to ### put commands into the queue. ### There is currently a timeout built into it that will cause ### the threads to die if they have no tasks after 10 seconds. self.process_launcher = apThread.ProcessLauncher(2, self.params['rundir']) ### write function log self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet)) ### upload command line parameters to database self.uploadScriptData() ### any custom init functions go here self.onInit()