def __init__(self,
optargs=sys.argv[1:],
quiet=False,
useglobalparams=True,
maxnproc=None):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.successful_run = False
self.params = {}
sys.stdout.write("\n\n")
self.quiet = quiet
self.maxnproc = maxnproc
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.timestamp = apParam.makeTimestamp()
self.argdict = {}
self.optdict = {}
apDisplay.printMsg("Time stamp: " + self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
apDisplay.printMsg("Function name: " + self.functionname)
self.appiondir = apParam.getAppionDirectory()
apDisplay.printMsg("Appion directory: " + self.appiondir)
self.parsePythonPath()
# loadavg = os.getloadavg()[0]
# if loadavg > 2.0:
# apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
# loadsquared = loadavg*loadavg
# time.sleep(loadavg)
# apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2)))
self.setLockname('lock')
### setup default parser: run directory, etc.
self.setParams(optargs, useglobalparams)
#if 'outdir' in self.params and self.params['outdir'] is not None:
# self.params['rundir'] = self.params['outdir']
self.checkConflicts()
if useglobalparams is True:
self.checkGlobalConflicts()
### setup run directory
self.setProcessingDirName()
self.setupRunDirectory()
### Start pool of threads to run subprocesses.
### Later you will use self.process_launcher.launch(...) to
### put commands into the queue.
### There is currently a timeout built into it that will cause
### the threads to die if they have no tasks after 10 seconds.
self.process_launcher = apThread.ProcessLauncher(
2, self.params['rundir'])
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### any custom init functions go here
self.onInit()
def __init__(self,optargs=sys.argv[1:],quiet=False):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.quiet = quiet
self.timestamp = apParam.makeTimestamp()
if not self.quiet:
apDisplay.printMsg("Time stamp: "+self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
if not self.quiet:
apDisplay.printMsg("Function name: "+self.functionname)
apParam.setUmask()
self.parsePythonPath()
loadavg = os.getloadavg()[0]
if loadavg > 2.0:
apDisplay.printMsg("Load average is %.2f, wait for %.1f second " % (round(loadavg,2),loadavg**2))
time.sleep(loadavg**2)
apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
### setup default parser: run directory, etc.
self.setParams(optargs)
self.checkConflicts()
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### any custom init functions go here
self.onInit()
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"):
"""
inputs:
aligned particle stack
number of classes
timestamp
output directory
output:
class averages
class variances
dendrogram.png
"""
if timestamp is None:
timestamp = apParam.makeTimestamp()
classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses))
classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses))
thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext)
### create class doc files
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
mySpider.toSpider(
"CL HE",
thresh,
spyder.fileFilter(dendrogramfile), # dendrogram doc file
rundir+"/classdoc_"+timestamp+"_****", # class doc file
)
### delete existing files
sys.stderr.write("delete existing files")
for dext in (".hed", ".img", dataext):
apFile.removeFile(classavg+dext)
apFile.removeFile(classvar+dext)
print ""
### create class averages
sys.stderr.write("create class averages")
for i in range(classes):
sys.stderr.write(".")
classnum = i+1
mySpider.toSpiderQuiet(
"AS R",
spyder.fileFilter(alignedstack)+"@******",
rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)),
"A",
(classavg+"@%04d" % (classnum)),
(classvar+"@%04d" % (classnum)),
)
mySpider.close()
print ""
### convert to IMAGIC
emancmd = "proc2d "+classavg+".spi "+classavg+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
emancmd = "proc2d "+classvar+".spi "+classvar+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return classavg,classvar
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"):
"""
inputs:
aligned particle stack
number of classes
timestamp
output directory
output:
class averages
class variances
dendrogram.png
"""
if timestamp is None:
timestamp = apParam.makeTimestamp()
classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses))
classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses))
thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext)
### create class doc files
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
mySpider.toSpider(
"CL HE",
thresh,
spyder.fileFilter(dendrogramfile), # dendrogram doc file
rundir+"/classdoc_"+timestamp+"_****", # class doc file
)
### delete existing files
sys.stderr.write("delete existing files")
for dext in (".hed", ".img", dataext):
apFile.removeFile(classavg+dext)
apFile.removeFile(classvar+dext)
print ""
### create class averages
sys.stderr.write("create %i class averages"%classes)
for i in range(classes):
sys.stderr.write(".")
classnum = i+1
mySpider.toSpiderQuiet(
"AS R",
spyder.fileFilter(alignedstack)+"@******",
rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)),
"A",
(classavg+"@%04d" % (classnum)),
(classvar+"@%04d" % (classnum)),
)
mySpider.close()
print ""
### convert to IMAGIC
emancmd = "proc2d "+classavg+".spi "+classavg+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
emancmd = "proc2d "+classvar+".spi "+classvar+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return classavg,classvar
def __init__(self,optargs=sys.argv[1:],quiet=False,useglobalparams=True,maxnproc=None):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.successful_run = False
self.params = {}
sys.stdout.write("\n\n")
self.quiet = quiet
self.maxnproc = maxnproc
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.timestamp = apParam.makeTimestamp()
self.argdict = {}
self.optdict = {}
apDisplay.printMsg("Time stamp: "+self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
apDisplay.printMsg("Function name: "+self.functionname)
self.appiondir = apParam.getAppionDirectory()
apDisplay.printMsg("Appion directory: "+self.appiondir)
self.parsePythonPath()
# loadavg = os.getloadavg()[0]
# if loadavg > 2.0:
# apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
# loadsquared = loadavg*loadavg
# time.sleep(loadavg)
# apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2)))
self.setLockname('lock')
### setup default parser: run directory, etc.
self.setParams(optargs,useglobalparams)
#if 'outdir' in self.params and self.params['outdir'] is not None:
# self.params['rundir'] = self.params['outdir']
self.checkConflicts()
if useglobalparams is True:
self.checkGlobalConflicts()
### setup run directory
self.setProcessingDirName()
self.setupRunDirectory()
### Start pool of threads to run subprocesses.
### Later you will use self.process_launcher.launch(...) to
### put commands into the queue.
### There is currently a timeout built into it that will cause
### the threads to die if they have no tasks after 10 seconds.
self.process_launcher = apThread.ProcessLauncher(2, self.params['rundir'])
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### any custom init functions go here
self.onInit()
def copyTemplatesToOutdir(params, timestamp=None):
newlist = []
for tmpl in params['templatelist']:
base = os.path.basename(tmpl)
old = os.path.abspath(tmpl)
### Rename file for new location
if timestamp is None:
timestamp = apParam.makeTimestamp()
#append the name of the directory to the filename
#basedir = os.path.split(os.path.dirname(old))[1]
#base = basedir+"_"+base
name, ext = os.path.splitext(base)
base = name + "-" + timestamp + ext
new = os.path.join(params['rundir'], base)
if os.path.isfile(new):
mdnew = apFile.md5sumfile(new)
mdold = apFile.md5sumfile(old)
if mdnew != mdold:
apDisplay.printError("a different template with name \'" +
new + "\' already exists!")
elif apDatabase.isTemplateInDB(mdnew):
apDisplay.printWarning("same template with md5sum \'" + mdnew +
"\' already exists in the DB!")
apDisplay.printMsg("skipping template \'" + old + "\'")
else:
apDisplay.printWarning("the same template with name \'" + new +
"\' already exists!")
newlist.append(base)
else:
#template is okay to copy and insert
apDisplay.printMsg("copying file " + old + " to " + new)
shutil.copyfile(old, new)
newlist.append(base)
#and only allow user read access just so they don't get deleted
#os.lchmod(new, 0666)
os.chmod(new, 0666)
params['templatelist'] = newlist
apDisplay.printColor("New template List:", "green")
pprint.pprint(params['templatelist'])
return
def copyTemplatesToOutdir(params, timestamp=None):
newlist = []
for tmpl in params['templatelist']:
base = os.path.basename(tmpl)
old = os.path.abspath(tmpl)
### Rename file for new location
if timestamp is None:
timestamp = apParam.makeTimestamp()
#append the name of the directory to the filename
#basedir = os.path.split(os.path.dirname(old))[1]
#base = basedir+"_"+base
name,ext = os.path.splitext(base)
base = name+"-"+timestamp+ext
new = os.path.join(params['rundir'], base)
if os.path.isfile(new):
mdnew = apFile.md5sumfile(new)
mdold = apFile.md5sumfile(old)
if mdnew != mdold:
apDisplay.printError("a different template with name \'"+new+"\' already exists!")
elif apDatabase.isTemplateInDB(mdnew):
apDisplay.printWarning("same template with md5sum \'"+mdnew+"\' already exists in the DB!")
apDisplay.printMsg("skipping template \'"+old+"\'")
else:
apDisplay.printWarning("the same template with name \'"+new+"\' already exists!")
newlist.append(base)
else:
#template is okay to copy and insert
apDisplay.printMsg("copying file "+old+" to "+new)
shutil.copyfile(old, new)
newlist.append(base)
#and only allow user read access just so they don't get deleted
#os.lchmod(new, 0666)
os.chmod(new, 0666)
params['templatelist'] = newlist
apDisplay.printColor("New template List:","green")
pprint.pprint(params['templatelist'])
return
def checkConflicts(self):
### unpickle results or parse logfile, set default parameters if missing
os.chdir(os.path.abspath(self.params['rundir']))
if os.path.isdir(os.path.join(self.params['rundir'], "recon")):
apDisplay.printWarning(
'recon dir already exist, no need to unpack')
else:
self.unpackResults()
''' These are required error checks, everything else should be possible to obtain from the timestamped pickle file '''
stackdata = apStack.getOnlyStackData(self.params['stackid'])
self.runparams = {}
self.runparams['original_apix'] = (
stackdata['pixelsize'] / 1e-10
) # convert pixelsize to angstroms per pixel.
self.runparams['original_boxsize'] = stackdata['boxsize']
# some functions in readRunParameters needs the above params
self.runparams.update(self.readRunParameters())
### parameters recovered from runparameter file(s)
if not self.runparams.has_key('stackid'):
if self.params['stackid'] is not None:
self.runparams['stackid'] = self.params['stackid']
else:
apDisplay.printError(
"stack id must be specified for proper database insertion")
if not self.runparams.has_key('modelid'):
if self.params['modelid'] is not None:
self.runparams['modelid'] = self.params['modelid']
else:
apDisplay.printError(
"model id must be specified for proper database insertion")
if not self.runparams.has_key('NumberOfReferences'):
if self.params['NumberOfReferences'] is not None:
self.runparams['NumberOfReferences'] = self.params[
'NumberOfReferences']
else:
apDisplay.printError(
"number of references produced during the refinement needs to be specified"
)
if not self.runparams.has_key('numiter'):
if self.params['numiter'] is not None:
self.runparams['numiter'] = self.params['numiter']
else:
apDisplay.printError(
"number of iterations run during the refinement needs to be specified"
)
if not self.runparams.has_key('boxsize'):
if self.params['boxsize'] is not None:
self.runparams['boxsize'] = self.params['boxsize']
else:
apDisplay.printError(
"boxsize of the map / particles submitted for refinement needs to be specified"
)
if not self.runparams.has_key('apix'):
if self.params['apix'] is not None:
self.runparams['apix'] = self.params['apix']
else:
apDisplay.printError(
"pixelsize of the map / particles submitted for refinement needs to be specified"
)
if not self.runparams.has_key('symmetry'):
if self.params['symid'] is not None:
self.runparams['symmetry'] = apSymmetry.getSymmetryDataFromID(
self.params['symid'])
else:
apDisplay.printError(
"symmetry ID must be specified, you can input --symid=25 for an asymmetric reconstruction"
)
# access multiModelRefinementRun this way in case it is not present
if 'multiModelRefinementRun' in vars(self):
if not self.runparams.has_key(
'NumberOfReferences'
) and self.multiModelRefinementRun is True:
if self.params['NumberOfReferences'] is not None:
self.runparams['NumberOfReferences'] = self.params[
'NumberOfReferences']
else:
apDisplay.printError(
"number of output models in refinement needs to be specified for multi-model run"
)
else:
if self.params['NumberOfReferences'] is not None:
self.runparams['NumberOfReferences'] = self.params[
'NumberOfReferences']
if self.runparams['NumberOfReferences'] > 1:
self.multiModelRefinementRun = True
else:
self.multiModelRefinementRun = False
else:
apDisplay.printError(
"number of output models (references) in refinement needs to be specified for multi-model run"
)
if not self.runparams.has_key('rundir'):
self.runparams['rundir'] = self.params['rundir']
if not self.runparams.has_key('reconstruction_working_dir'):
self.runparams['reconstruction_working_dir'] = str(
self.package) + "_results"
if not self.runparams.has_key('mask'):
self.runparams['mask'] = None
if not self.runparams.has_key('imask'):
self.runparams['imask'] = None
if not self.runparams.has_key('alignmentInnerRadius'):
self.runparams['alignmentInnerRadius'] = None
if not self.runparams.has_key('alignmentOuterRadius'):
self.runparams['alignmentOuterRadius'] = None
if not self.runparams.has_key('angularSamplingRate'):
self.runparams['angularSamplingRate'] = None
### parameters specified for upload
if self.params['jobid'] is not None:
self.params['jobinfo'] = appiondata.ApAppionJobData.direct_query(
self.params['jobid'])
else:
jobid = self.tryToGetJobID()
if jobid is not False:
self.params[
'jobinfo'] = appiondata.ApAppionJobData.direct_query(
self.params['jobid'])
else:
self.params['jobinfo'] = None
if self.params[
'timestamp'] is None and self.package == "external_package":
apDisplay.printError("a timestamp (or some identifier) must be specified with the files. For example, the 3D mrc file " \
"for iteration 1 should be named 'recon_YourIdentifier_it001_vol001.mrc, in which case the timestamp should be specified " \
"as --timestamp=YourIdentifier")
elif self.params['timestamp'] is None and self.params[
'jobid'] is None and self.package != "external_package":
self.params['timestamp'] = apParam.makeTimestamp()
elif self.params['timestamp'] is None and self.params[
'jobid'] is not None:
timestamp = self.getTimestamp()
if timestamp is None:
self.params['timestamp'] = apParam.makeTimestamp()
# apDisplay.printError("please specify the timestamp associated with the refinement parameters, e.g. --timestamp=08nov25c07")
### basic refinement variables
self.initializeRefinementUploadVariables()
return
def kmeansCluster(alignedstack, numpart=None, numclasses=40, timestamp=None,
factorlist=range(1,5), corandata="coran/corandata", dataext=".spi"):
"""
inputs:
outputs:
"""
if timestamp is None:
timestamp = apParam.makeTimestamp()
if alignedstack[-4:] == dataext:
alignedstack = alignedstack[:-4]
rundir = "cluster"
classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses))
classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses))
apParam.createDirectory(rundir)
for i in range(numclasses):
apFile.removeFile(rundir+("/classdoc%04d" % (i+1))+dataext)
apFile.removeFile(rundir+("/allclassesdoc%04d" % (numclasses))+dataext)
### make list of factors
factorstr, factorkey = operations.intListToString(factorlist)
### do k-means clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
mySpider.toSpider(
"CL KM",
corandata+"_IMC", # path to coran data
str(numclasses), # num classes
factorstr, # factor string
)
## weight for each factor
for fact in factorlist:
mySpider.toSpiderQuiet("1.0")
randnum = (int(random.random()*1000) + 1)
mySpider.toSpider(
str(randnum),
rundir+"/classdoc_"+timestamp+"_****", # class doc file
rundir+("/allclassesdoc%04d" % (numclasses)), #clusterdoc file
)
mySpider.close()
### delete existing files
sys.stderr.write("delete existing files")
for dext in (".hed", ".img", dataext):
apFile.removeFile(classavg+dext)
apFile.removeFile(classvar+dext)
print ""
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
### create class averages
apDisplay.printMsg("Averaging particles into classes")
for i in range(numclasses):
classnum = i+1
mySpider.toSpiderQuiet(
"AS R",
spyder.fileFilter(alignedstack)+"@******",
rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)),
"A",
(classavg+"@%04d" % (classnum)),
(classvar+"@%04d" % (classnum)),
)
if classnum % 10 == 0:
sys.stderr.write(".")
time.sleep(1)
mySpider.close()
### convert to IMAGIC
emancmd = "proc2d "+classavg+".spi "+classavg+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
emancmd = "proc2d "+classvar+".spi "+classvar+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return classavg,classvar
def checkConflicts(self):
### unpickle results or parse logfile, set default parameters if missing
os.chdir(os.path.abspath(self.params["rundir"]))
if os.path.isdir(os.path.join(self.params["rundir"], "recon")):
apDisplay.printWarning("recon dir already exist, no need to unpack")
else:
self.unpackResults()
""" These are required error checks, everything else should be possible to obtain from the timestamped pickle file """
stackdata = apStack.getOnlyStackData(self.params["stackid"])
self.runparams = {}
self.runparams["original_apix"] = stackdata["pixelsize"] / 1e-10 # convert pixelsize to angstroms per pixel.
self.runparams["original_boxsize"] = stackdata["boxsize"]
# some functions in readRunParameters needs the above params
self.runparams.update(self.readRunParameters())
### parameters recovered from runparameter file(s)
if not self.runparams.has_key("stackid"):
if self.params["stackid"] is not None:
self.runparams["stackid"] = self.params["stackid"]
else:
apDisplay.printError("stack id must be specified for proper database insertion")
if not self.runparams.has_key("modelid"):
if self.params["modelid"] is not None:
self.runparams["modelid"] = self.params["modelid"]
else:
apDisplay.printError("model id must be specified for proper database insertion")
if not self.runparams.has_key("NumberOfReferences"):
if self.params["NumberOfReferences"] is not None:
self.runparams["NumberOfReferences"] = self.params["NumberOfReferences"]
else:
apDisplay.printError("number of references produced during the refinement needs to be specified")
if not self.runparams.has_key("numiter"):
if self.params["numiter"] is not None:
self.runparams["numiter"] = self.params["numiter"]
else:
apDisplay.printError("number of iterations run during the refinement needs to be specified")
if not self.runparams.has_key("boxsize"):
if self.params["boxsize"] is not None:
self.runparams["boxsize"] = self.params["boxsize"]
else:
apDisplay.printError("boxsize of the map / particles submitted for refinement needs to be specified")
if not self.runparams.has_key("apix"):
if self.params["apix"] is not None:
self.runparams["apix"] = self.params["apix"]
else:
apDisplay.printError("pixelsize of the map / particles submitted for refinement needs to be specified")
if not self.runparams.has_key("symmetry"):
if self.params["symid"] is not None:
self.runparams["symmetry"] = apSymmetry.getSymmetryDataFromID(self.params["symid"])
else:
apDisplay.printError(
"symmetry ID must be specified, you can input --symid=25 for an asymmetric reconstruction"
)
# access multiModelRefinementRun this way in case it is not present
if "multiModelRefinementRun" in vars(self):
if not self.runparams.has_key("NumberOfReferences") and self.multiModelRefinementRun is True:
if self.params["NumberOfReferences"] is not None:
self.runparams["NumberOfReferences"] = self.params["NumberOfReferences"]
else:
apDisplay.printError(
"number of output models in refinement needs to be specified for multi-model run"
)
else:
if self.params["NumberOfReferences"] is not None:
self.runparams["NumberOfReferences"] = self.params["NumberOfReferences"]
if self.runparams["NumberOfReferences"] > 1:
self.multiModelRefinementRun = True
else:
self.multiModelRefinementRun = False
else:
apDisplay.printError(
"number of output models (references) in refinement needs to be specified for multi-model run"
)
if not self.runparams.has_key("rundir"):
self.runparams["rundir"] = self.params["rundir"]
if not self.runparams.has_key("reconstruction_working_dir"):
self.runparams["reconstruction_working_dir"] = str(self.package) + "_results"
if not self.runparams.has_key("mask"):
self.runparams["mask"] = None
if not self.runparams.has_key("imask"):
self.runparams["imask"] = None
if not self.runparams.has_key("alignmentInnerRadius"):
self.runparams["alignmentInnerRadius"] = None
if not self.runparams.has_key("alignmentOuterRadius"):
self.runparams["alignmentOuterRadius"] = None
if not self.runparams.has_key("angularSamplingRate"):
self.runparams["angularSamplingRate"] = None
### parameters specified for upload
if self.params["jobid"] is not None:
self.params["jobinfo"] = appiondata.ApAppionJobData.direct_query(self.params["jobid"])
else:
jobid = self.tryToGetJobID()
if jobid is not False:
self.params["jobinfo"] = appiondata.ApAppionJobData.direct_query(self.params["jobid"])
else:
self.params["jobinfo"] = None
if self.params["timestamp"] is None and self.package == "external_package":
apDisplay.printError(
"a timestamp (or some identifier) must be specified with the files. For example, the 3D mrc file "
"for iteration 1 should be named 'recon_YourIdentifier_it001_vol001.mrc, in which case the timestamp should be specified "
"as --timestamp=YourIdentifier"
)
elif self.params["timestamp"] is None and self.params["jobid"] is None and self.package != "external_package":
self.params["timestamp"] = apParam.makeTimestamp()
elif self.params["timestamp"] is None and self.params["jobid"] is not None:
timestamp = self.getTimestamp()
if timestamp is None:
self.params["timestamp"] = apParam.makeTimestamp()
# apDisplay.printError("please specify the timestamp associated with the refinement parameters, e.g. --timestamp=08nov25c07")
### basic refinement variables
self.initializeRefinementUploadVariables()
return
def createProjections(self, pad=False, invert=False):
timestamp = apParam.makeTimestamp()
eulerlist = self.setEulers()
eulerfile = os.path.join(self.params['rundir'], "eulers.lst")
f = open(eulerfile, "w")
projcount = numpy.zeros((len(eulerlist)), dtype=numpy.uint16)
angsum = numpy.zeros((len(eulerlist),3), dtype=numpy.float32)
t0 = time.time()
for i in range(self.params['projcount']):
projnum = int(random.random()*len(eulerlist))
alt = random.gauss(eulerlist[projnum][0], self.params['projstdev'])
az = random.gauss(eulerlist[projnum][1], self.params['projstdev'])
#phi = random.random()*360.0-180.0
#phi = random.random()*360.0
if self.params['rotang'] != 0:
phi = random.uniform(-1*self.params['rotang'], self.params['rotang'])
else:
phi = 0.0
f.write("%.8f\t%.8f\t%.8f\n"%(alt,az,phi))
### stats
projcount[projnum] += 1
angsum[projnum,0] += alt
angsum[projnum,1] += az
angsum[projnum,2] += phi
apDisplay.printMsg("Finished random in %s, %.3f ns per iteration"
%(apDisplay.timeString(time.time()-t0), 1.0e6 * (time.time()-t0)/float(self.params['projcount'])))
f.close()
print "projection count", projcount
for i in range(len(eulerlist)):
angavg = angsum[i,:]/projcount[i]
print "angle average %d: %03.3f, %03.3f, %03.3f"%(i, angavg[0], angavg[1], angavg[2])
### first get rid of projection artifacts from insufficient padding
if self.params['threedfile'] is not None:
origfile = self.params['threedfile']
elif self.params['modelid'] is not None:
self.modelparams = appiondata.ApInitialModelData.direct_query(self.params['modelid'])
origfile = os.path.join(self.modelparams['path']['path'], self.modelparams['name'])
if self.params['apix'] is None:
self.params['apix'] = self.modelparams['pixelsize']
if self.params['box'] is None:
self.params['box'] = self.modelparams['boxsize']
clipped = os.path.join(self.params['rundir'], "clipped.mrc")
newsize = self.params['box'] * 1.5
emancmd = "proc3d "+origfile+" "+clipped+" clip="+str(int(newsize))+","+str(int(newsize))+","+str(int(newsize))+" edgenorm"
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
### project resized file
filename = os.path.join(self.params['rundir'], 'proj.img')
apFile.removeStack(filename)
emancmd = "project3d "+clipped+" out="+filename+" list="+eulerfile
t0 = time.time()
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
apDisplay.printMsg("Finished project3d in %s, %.3f ms per iteration"
%(apDisplay.timeString(time.time()-t0), 1.0e3 * (time.time()-t0)/float(self.params['projcount'])))
### post-process stack
self.post_process_stack(filename, pad, invert)
return filename
def kmeansCluster(alignedstack,
numpart=None,
numclasses=40,
timestamp=None,
factorlist=range(1, 5),
corandata="coran/corandata",
dataext=".spi"):
"""
inputs:
outputs:
"""
if timestamp is None:
timestamp = apParam.makeTimestamp()
if alignedstack[-4:] == dataext:
alignedstack = alignedstack[:-4]
rundir = "cluster"
classavg = rundir + "/" + ("classavgstack_%s_%03d" %
(timestamp, numclasses))
classvar = rundir + "/" + ("classvarstack_%s_%03d" %
(timestamp, numclasses))
apParam.createDirectory(rundir)
for i in range(numclasses):
apFile.removeFile(rundir + ("/classdoc%04d" % (i + 1)) + dataext)
apFile.removeFile(rundir + ("/allclassesdoc%04d" % (numclasses)) + dataext)
### make list of factors
factorstr, factorkey = operations.intListToString(factorlist)
### do k-means clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
mySpider.toSpider(
"CL KM",
corandata + "_IMC", # path to coran data
str(numclasses), # num classes
factorstr, # factor string
)
## weight for each factor
for fact in factorlist:
mySpider.toSpiderQuiet("1.0")
randnum = (int(random.random() * 1000) + 1)
mySpider.toSpider(
str(randnum),
rundir + "/classdoc_" + timestamp + "_****", # class doc file
rundir + ("/allclassesdoc%04d" % (numclasses)), #clusterdoc file
)
mySpider.close()
### delete existing files
sys.stderr.write("delete existing files")
for dext in (".hed", ".img", dataext):
apFile.removeFile(classavg + dext)
apFile.removeFile(classvar + dext)
print ""
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
### create class averages
apDisplay.printMsg("Averaging particles into classes")
for i in range(numclasses):
classnum = i + 1
mySpider.toSpiderQuiet(
"AS R",
spyder.fileFilter(alignedstack) + "@******",
rundir + ("/classdoc_" + timestamp + "_%04d" % (classnum)),
"A",
(classavg + "@%04d" % (classnum)),
(classvar + "@%04d" % (classnum)),
)
if classnum % 10 == 0:
sys.stderr.write(".")
time.sleep(1)
mySpider.close()
### convert to IMAGIC
emancmd = "proc2d " + classavg + ".spi " + classavg + ".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
emancmd = "proc2d " + classvar + ".spi " + classvar + ".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return classavg, classvar
def __init__(self,
optargs=sys.argv[1:],
quiet=False,
useglobalparams=True,
maxnproc=None):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.successful_run = False
self.clusterjobdata = None
self.params = {}
sys.stdout.write("\n\n")
self.quiet = quiet
self.maxnproc = maxnproc
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.timestamp = apParam.makeTimestamp()
self.argdict = {}
self.optdict = {}
apDisplay.printMsg("Time stamp: " + self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
apDisplay.printMsg("Function name: " + self.functionname)
self.appiondir = apParam.getAppionDirectory()
apDisplay.printMsg("Appion directory: " + self.appiondir)
self.parsePythonPath()
loadavg = os.getloadavg()[0]
if loadavg > 2.0:
apDisplay.printMsg("Load average is high " +
str(round(loadavg, 2)))
loadsquared = loadavg * loadavg
time.sleep(loadavg)
apDisplay.printMsg("New load average " +
str(round(os.getloadavg()[0], 2)))
self.setLockname('lock')
### setup default parser: run directory, etc.
self.setParams(optargs, useglobalparams)
#if 'outdir' in self.params and self.params['outdir'] is not None:
# self.params['rundir'] = self.params['outdir']
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(
self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '" + newdbname + "'",
"green")
### check if user wants to print help message
if 'commit' in self.params and self.params['commit'] is True:
apDisplay.printMsg("Committing data to database")
else:
apDisplay.printWarning("Not committing data to database")
self.checkConflicts()
if useglobalparams is True:
self.checkGlobalConflicts()
### setup run directory
self.setProcessingDirName()
self.setupRunDirectory()
### Start pool of threads to run subprocesses.
### Later you will use self.process_launcher.launch(...) to
### put commands into the queue.
### There is currently a timeout built into it that will cause
### the threads to die if they have no tasks after 10 seconds.
self.process_launcher = apThread.ProcessLauncher(
2, self.params['rundir'])
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### upload command line parameters to database
self.uploadScriptData()
### any custom init functions go here
self.onInit()
def createProjections(self, pad=False, invert=False):
timestamp = apParam.makeTimestamp()
eulerlist = self.setEulers()
eulerfile = os.path.join(self.params['rundir'], "eulers.lst")
f = open(eulerfile, "w")
projcount = numpy.zeros((len(eulerlist)), dtype=numpy.uint16)
angsum = numpy.zeros((len(eulerlist),3), dtype=numpy.float32)
t0 = time.time()
for i in range(self.params['projcount']):
projnum = int(random.random()*len(eulerlist))
alt = random.gauss(eulerlist[projnum][0], self.params['projstdev'])
az = random.gauss(eulerlist[projnum][1], self.params['projstdev'])
#phi = random.random()*360.0-180.0
#phi = random.random()*360.0
if self.params['rotang'] != 0:
phi = random.uniform(-1*self.params['rotang'], self.params['rotang'])
else:
phi = 0.0
f.write("%.8f\t%.8f\t%.8f\n"%(alt,az,phi))
### stats
projcount[projnum] += 1
angsum[projnum,0] += alt
angsum[projnum,1] += az
angsum[projnum,2] += phi
apDisplay.printMsg("Finished random in %s, %.3f ns per iteration"
%(apDisplay.timeString(time.time()-t0), 1.0e6 * (time.time()-t0)/float(self.params['projcount'])))
f.close()
print "projection count", projcount
for i in range(len(eulerlist)):
angavg = angsum[i,:]/projcount[i]
print "angle average %d: %03.3f, %03.3f, %03.3f"%(i, angavg[0], angavg[1], angavg[2])
### first get rid of projection artifacts from insufficient padding
if self.params['threedfile'] is not None:
origfile = self.params['threedfile']
elif self.params['modelid'] is not None:
self.modelparams = appiondata.ApInitialModelData.direct_query(self.params['modelid'])
origfile = os.path.join(self.modelparams['path']['path'], self.modelparams['name'])
if self.params['apix'] is None:
self.params['apix'] = self.modelparams['pixelsize']
if self.params['box'] is None:
self.params['box'] = self.modelparams['boxsize']
clipped = os.path.join(self.params['rundir'], "clipped.mrc")
newsize = self.params['box'] * 1.5
emancmd = "proc3d "+origfile+" "+clipped+" clip="+str(int(newsize))+","+str(int(newsize))+","+str(int(newsize))+" edgenorm"
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
### project resized file
filename = os.path.join(self.params['rundir'], 'proj.img')
apFile.removeStack(filename)
emancmd = "project3d "+clipped+" out="+filename+" list="+eulerfile
t0 = time.time()
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
apDisplay.printMsg("Finished project3d in %s, %.3f ms per iteration"
%(apDisplay.timeString(time.time()-t0), 1.0e3 * (time.time()-t0)/float(self.params['projcount'])))
### post-process stack
self.post_process_stack(filename, pad, invert)
return filename
def __init__(self,optargs=sys.argv[1:],quiet=False,useglobalparams=True,maxnproc=None):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.successful_run = False
self.clusterjobdata = None
self.params = {}
sys.stdout.write("\n\n")
self.quiet = quiet
self.maxnproc = maxnproc
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.timestamp = apParam.makeTimestamp()
self.argdict = {}
self.optdict = {}
apDisplay.printMsg("Time stamp: "+self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
apDisplay.printMsg("Function name: "+self.functionname)
self.appiondir = apParam.getAppionDirectory()
apDisplay.printMsg("Appion directory: "+self.appiondir)
hostname = apParam.getHostname()
apDisplay.printMsg("Processing hostname: "+hostname)
self.parsePythonPath()
# loadavg = os.getloadavg()[0]
# if loadavg > 2.0:
# apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
# loadsquared = loadavg*loadavg
# time.sleep(loadavg)
# apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2)))
self.setLockname('lock')
### setup default parser: run directory, etc.
self.setParams(optargs,useglobalparams)
#if 'outdir' in self.params and self.params['outdir'] is not None:
# self.params['rundir'] = self.params['outdir']
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
### check if user wants to print help message
if 'commit' in self.params and self.params['commit'] is True:
apDisplay.printMsg("Committing data to database")
else:
apDisplay.printWarning("Not committing data to database")
self.checkConflicts()
if useglobalparams is True:
self.checkGlobalConflicts()
### setup run directory
self.setProcessingDirName()
self.setupRunDirectory()
### Start pool of threads to run subprocesses.
### Later you will use self.process_launcher.launch(...) to
### put commands into the queue.
### There is currently a timeout built into it that will cause
### the threads to die if they have no tasks after 10 seconds.
self.process_launcher = apThread.ProcessLauncher(2, self.params['rundir'])
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### upload command line parameters to database
self.uploadScriptData()
### any custom init functions go here
self.onInit()