def peakTreeToPikFile(peaktree, imgname, tmpl, rundir="."):
outpath = os.path.join(rundir, "pikfiles")
apParam.createDirectory(outpath, warning=False)
outfile = os.path.join(outpath, imgname+"."+str(tmpl)+".pik")
apFile.removeFile(outfile, warn=True)
#WRITE PIK FILE
f=open(outfile, 'w')
f.write("#filename x y mean stdev corr_coeff peak_size templ_num angle moment diam\n")
for peakdict in peaktree:
row = peakdict['ycoord']
col = peakdict['xcoord']
if 'corrcoeff' in peakdict:
rho = peakdict['corrcoeff']
else:
rho = 1.0
size = peakdict['peakarea']
mean_str = "%.4f" % peakdict['correlation']
std_str = "%.4f" % peakdict['peakstddev']
mom_str = "%.4f" % peakdict['peakmoment']
if peakdict['diameter'] is not None:
diam_str = "%.2f" % peakdict['diameter']
else:
diam_str = "0"
if 'template' in peakdict:
tmplnum = peakdict['template']
else:
tmplnum = tmpl
#filename x y mean stdev corr_coeff peak_size templ_num angle moment
out = imgname+".mrc "+str(int(col))+" "+str(int(row))+ \
" "+mean_str+" "+std_str+" "+str(rho)+" "+str(int(size))+ \
" "+str(tmplnum)+" 0 "+mom_str+" "+diam_str
f.write(str(out)+"\n")
f.close()
def createPeakJpeg(imgdata, peaktree, params, procimgarray=None): if 'templatelist' in params: count = len(params['templatelist']) else: count = 1 bin = int(params["bin"]) diam = float(params["diam"]) apix = float(params["apix"]) binpixrad = diam/apix/2.0/float(bin) imgname = imgdata['filename'] jpegdir = os.path.join(params['rundir'],"jpgs") apParam.createDirectory(jpegdir, warning=False) if params['uncorrected']: imgarray = apImage.correctImage(imgdata, params) else: imgarray = imgdata['image'] if procimgarray is not None: #instead of re-processing image use one that is already processed... imgarray = procimgarray else: imgarray = apImage.preProcessImage(imgarray, bin=bin, planeReg=False, params=params) outfile = os.path.join(jpegdir, imgname+".prtl.jpg") msg = not params['background'] subCreatePeakJpeg(imgarray, peaktree, binpixrad, outfile, bin, msg) return
def ClCla(alignedstack, numpart=None, numclasses=40,
factorlist=range(1,5), corandata="coran/corandata", dataext=".spi"):
"""
this doesn't work
"""
if alignedstack[-4:] == dataext:
alignedstack = alignedstack[:-4]
rundir = "cluster"
classavg = rundir+"/"+("classavgstack%03d" % numclasses)
classvar = rundir+"/"+("classvarstack%03d" % numclasses)
apParam.createDirectory(rundir)
for i in range(numclasses):
apFile.removeFile(rundir+("/classdoc%04d" % (i+1))+dataext)
apFile.removeFile(rundir+"/clusterdoc"+dataext)
factorstr, factorkey = operations.intListToString(factorlist)
### do hierarchical clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider(
"CL CLA",
corandata, # path to coran data
rundir+"/clusterdoc", #clusterdoc file
factorstr, #factor numbers
"5,8",
"4",
"2", # minimum number of particles per class
"Y", rundir+"/dendrogram.ps",
"Y", rundir+"/dendrogramdoc",
)
mySpider.close()
def preLoopFunctions(self): self.powerspecdir = os.path.join(self.params['rundir'], "opimages") apParam.createDirectory(self.powerspecdir, warning=False) self.ctfrun = None self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) return
def processStack(self, stackarray):
self.dirnum += 1
self.partdir = os.path.join(self.rootdir, "%03d" % (self.dirnum))
apParam.createDirectory(self.partdir, warning=False)
for partarray in stackarray:
self.processParticle(partarray)
self.index += 1 #you must have this line in your loop
def createPeakJpeg(imgdata, peaktree, params, procimgarray=None):
if 'templatelist' in params:
count = len(params['templatelist'])
else:
count = 1
bin = int(params["bin"])
diam = float(params["diam"])
apix = float(params["apix"])
binpixrad = diam / apix / 2.0 / float(bin)
imgname = imgdata['filename']
jpegdir = os.path.join(params['rundir'], "jpgs")
apParam.createDirectory(jpegdir, warning=False)
if params['uncorrected']:
imgarray = apImage.correctImage(imgdata, params)
else:
imgarray = imgdata['image']
if procimgarray is not None:
#instead of re-processing image use one that is already processed...
imgarray = procimgarray
else:
imgarray = apImage.preProcessImage(imgarray,
bin=bin,
planeReg=False,
params=params)
outfile = os.path.join(jpegdir, imgname + ".prtl.jpg")
msg = not params['background']
subCreatePeakJpeg(imgarray, peaktree, binpixrad, outfile, bin, msg)
return
def preLoopFunctions(self):
apParam.createDirectory(os.path.join(self.params['rundir'],
"pikfiles"),
warning=False)
if self.params['sessionname'] is not None:
self.processAndSaveAllImages()
## use labels from params if specified
if self.params['labels']:
self.labels = self.params['labels']
else:
self.labels = []
### this is confusing and needs someone to look at it.
if self.params['helicalstep']:
self.labels = ['User', 'Helical']
if self.params['helix'] is True:
self.labels = ['Add Helix', 'Stored Helices']
## If no labels specified or previous picks to get labels from,
## then use default label 'particle'.
if not self.labels:
self.labels = ['particle_w/o_label']
self.setApp()
self.app.appionloop = self
self.threadJpeg = True
def createTemplateStack(self):
"""
takes the spider file and creates an average template of all particles
"""
templatestack = os.path.join(self.params['rundir'], "templatestack00.spi")
apFile.removeFile(templatestack, warn=True)
### hack to use standard filtering library
templateparams = {}
templateparams['apix'] = self.stack['apix']
templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
templateparams['templateIds'] = self.templatelist
templateparams['bin'] = self.params['bin']
templateparams['lowpass'] = self.params['lowpass']
templateparams['median'] = None
templateparams['pixlimit'] = None
print templateparams
apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
filelist = apTemplate.getTemplates(templateparams)
for mrcfile in filelist:
emancmd = ("proc2d templates/"+mrcfile+" "+templatestack
+" clip="+str(self.boxsize)+","+str(self.boxsize)
+" spider ")
if self.params['inverttemplates'] is True:
emancmd += " invert "
apEMAN.executeEmanCmd(emancmd, showcmd=False)
return templatestack
def makeStackInFormatFromDefault(stackdata, format='eman'):
'''
Function to create the stack in the requested format
Current default is Imagic format used in EMAN
'''
if format not in formats:
apDisplay.printError('Unknown stack format %s requested' % format)
stackroot, instackdir, outstackdir = getFormattedStackInfo(
stackdata, format)
instackpath = os.path.join(instackdir, stackroot + '.hed')
if format == 'eman':
return stackdata['path']['path']
apParam.createDirectory(outstackdir)
rundir = os.getcwd()
os.chdir(outstackdir)
if format == 'spider':
outstackpath = os.path.join(outstackdir,
stackroot + stack_exts[format][0])
emancmd = "proc2d %s %s spidersingle" % (instackpath, outstackpath)
apEMAN.executeEmanCmd(emancmd, showcmd=True)
if format == 'frealign':
outstackpath = os.path.join(outstackdir,
stackroot + stack_exts[format][0])
emancmd = "proc2d %s %s invert" % (instackpath, outstackpath)
apEMAN.executeEmanCmd(emancmd, showcmd=True)
if format == 'xmipp':
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(instackpath)
os.chdir(rundir)
return outstackdir
def createTemplateStack(self):
"""
convert spider template stack
"""
templatestack = os.path.join(self.params['rundir'], "templatestack.spi")
apFile.removeFile(templatestack, warn=True)
### hack to use standard filtering library
templateparams = {}
templateparams['apix'] = self.stack['apix']
templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
templateparams['templateIds'] = self.templatelist
templateparams['bin'] = self.params['bin']
templateparams['lowpass'] = self.params['lowpass']
templateparams['median'] = None
templateparams['pixlimit'] = None
print 'Converting reference templates:\n', templateparams
apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
filelist = apTemplate.getTemplates(templateparams)
localclip = self.clipsize/self.params['bin']
for mrcfile in filelist:
emancmd = ("proc2d templates/"+mrcfile+" "+templatestack
+" clip="+str(localclip)+","+str(localclip)
+" edgenorm spiderswap ")
if self.params['inverttemplates'] is True:
emancmd += " invert "
apEMAN.executeEmanCmd(emancmd, showcmd=False)
return templatestack
def createOrientationReference(self):
"""
convert spider orientation reference
"""
orientref = os.path.join(self.params['rundir'], "orient.spi")
apFile.removeFile(orientref, warn=True)
#templatedata = apTemplate.getTemplateFromId(self.params['orientref'])
#templatefile = os.path.join(templatedata['path']['path'], templatedata['templatename'])
### hack to use standard filtering library
templateparams = {}
templateparams['apix'] = self.stack['apix']
templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
templateparams['templateIds'] = [self.params['orientref'],]
templateparams['bin'] = self.params['bin']
templateparams['lowpass'] = self.params['lowpass']
templateparams['median'] = None
templateparams['pixlimit'] = None
print 'Converting orientation reference:\n', templateparams
apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
filelist = apTemplate.getTemplates(templateparams)
mrcfile = filelist[0]
localclip = self.clipsize/self.params['bin']
emancmd = ("proc2d templates/"+mrcfile+" "+orientref
+" clip="+str(localclip)+","+str(localclip)
+" edgenorm spiderswap-single ")
if self.params['inverttemplates'] is True:
emancmd += " invert "
apEMAN.executeEmanCmd(emancmd, showcmd=False)
return orientref
def runAppionScriptInSubprocess(self, cmd, logfilepath):
# Running another AppionScript as a subprocess
apDisplay.printMsg('running AppionScript:')
apDisplay.printMsg('------------------------------------------------')
apDisplay.printMsg(cmd)
# stderr=subprocess.PIPE only works with shell=True with python 2.4.
# works on python 2.6. Use shell=True now but shell=True does not
# work with path changed by appionwrapper. It behaves as if the wrapper
# is not used
proc = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout_value = proc.communicate()[0]
while proc.returncode is None:
time.wait(60)
stdout_value = proc.communicate()[0]
try:
logdir = os.path.dirname(logfilepath)
apParam.createDirectory(logdir)
file = open(logfilepath, 'w')
except:
apDisplay.printError(
'Log file can not be created, process did not run.')
file.write(stdout_value)
file.close()
if proc.returncode > 0:
pieces = cmd.split(' ')
apDisplay.printWarning(
'AppionScript %s had an error. Please check its log file: \n%s'
% (pieces[0].upper(), logfilepath))
else:
apDisplay.printMsg('AppionScript ran successfully')
apDisplay.printMsg('------------------------------------------------')
return proc.returncode
def checkAndCopyFiles(self):
if not os.path.isfile(self.params['inputstack']):
apDisplay.printError("Stack file %s does not exist"%(self.params['inputstack']))
if not os.path.isfile(self.params['inputvol']):
apDisplay.printError("Initial model file %s does not exist"%(self.params['inputvol']))
if not os.path.isfile(self.params['inputparam']):
apDisplay.printError("Initial parameter file %s does not exist"%(self.params['inputparam']))
apParam.createDirectory(self.params['rundir'])
firstVolFile = os.path.join(self.params['rundir'], "initial_volume.mrc")
apDisplay.printColor("Linking initial model %s to recon folder %s"%
(self.params['inputvol'], firstVolFile), "purple")
if not os.path.exists(firstVolFile):
os.symlink(self.params['inputvol'], firstVolFile)
#shutil.copy(self.params['inputvol'], firstVolFile)
firstParamFile = os.path.join(self.params['rundir'], "initial_params.par")
apDisplay.printColor("Copying initial parameters %s to recon folder %s"
%(self.params['inputparam'], firstParamFile), "purple")
apFile.removeFile(firstParamFile)
time.sleep(0.1)
shutil.copy(self.params['inputparam'], firstParamFile)
os.chdir(self.params['rundir'])
for side in ('left', 'right'):
prefix = "%s.iter%02d"%(side, 0)
volFile = "threed.%s.mrc"%(prefix)
if not os.path.islink(volFile):
os.symlink(os.path.basename(firstVolFile), volFile)
self.createSplitParamFiles(firstParamFile)
return
def makeRefineTasks(self,iter):
tasks = {}
nproc = self.params['nproc']
iterpath = os.path.join(self.params['recondir'],'iter%03d' % (iter))
apParam.createDirectory(iterpath, warning=False)
# set frealign param keys
frealign_param_keys = self.setFrealignRefineParams()
inputlines_template = self.createFrealignInputLineTemplate(iter,frealign_param_keys)
if not self.params['recononly'] and iter > 0:
# refine parallelizable to multiple nodes
refine_files = self.writeMultipleRefineShells(iter,inputlines_template,iterpath,self.nproc)
# use mpi to parallelize the processes
masterfile_name = 'mpi.iter%03d.run.sh' % (iter)
mpi_refine = self.setupMPIRun(refine_files,self.nproc,iterpath,masterfile_name)
tasks = self.addToTasks(tasks,mpi_refine,self.mem,self.ppn)
tasks = self.logTaskStatus(tasks,'refine',os.path.join(iterpath,'proc%03d/frealign.proc%03d.out' % (self.nproc-1,self.nproc-1)),iter)
if not self.params['refineonly'] or iter == 0:
# combine and recon parallelized only on one node
recon_file = self.writeReconShell(iter,inputlines_template,iterpath,self.ppn)
mp_recon = self.setupMPRun(recon_file,self.recon_mem,self.ppn)
tasks = self.addToTasks(tasks,mp_recon,self.recon_mem,self.ppn)
tasks = self.addToTasks(tasks,'cd %s' % iterpath)
tasks = self.addToTasks(tasks,'getRes.pl %s %d %.3f >> ../resolution.txt' % (iter,self.params['boxsize'],self.params['apix']))
tasks = self.logTaskStatus(tasks,'eotest','../resolution.txt',iter)
tasks = self.addToTasks(tasks,'cd %s' % self.params['recondir'])
tasks = self.logTaskStatus(tasks,'recon',os.path.join(iterpath,'frealign.recon.out'),iter)
else:
recon_file = self.writeRefineOnlyCombineShell(iter,iterpath)
mp_recon = self.setupMPRun(recon_file,2,1)
tasks = self.addToTasks(tasks,mp_recon,2,1)
tasks = self.addToTasks(tasks,'cd %s' % iterpath)
return tasks
def createTemplateStack(self):
"""
takes the spider file and creates an average template of all particles
"""
templatestack = os.path.join(self.params['rundir'], "templatestack00.spi")
apFile.removeFile(templatestack, warn=True)
### hack to use standard filtering library
templateparams = {}
templateparams['apix'] = self.stack['apix']
templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
templateparams['templateIds'] = self.templatelist
templateparams['bin'] = self.params['bin']
templateparams['lowpass'] = self.params['lowpass']
templateparams['median'] = None
templateparams['pixlimit'] = None
print templateparams
apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
filelist = apTemplate.getTemplates(templateparams)
for mrcfile in filelist:
emancmd = ("proc2d templates/"+mrcfile+" "+templatestack
+" clip="+str(self.boxsize)+","+str(self.boxsize)
+" spider ")
if self.params['inverttemplates'] is True:
emancmd += " invert "
apEMAN.executeEmanCmd(emancmd, showcmd=False)
return templatestack
def runFileScriptInSubprocess(self,cmd,logfilepath):
# Running another FileScript as a subprocess
apDisplay.printMsg('running FileScript:')
apDisplay.printMsg('------------------------------------------------')
apDisplay.printMsg(cmd)
# stderr=subprocess.PIPE only works with shell=True with python 2.4.
# works on python 2.6. Use shell=True now but shell=True does not
# work with path changed by appionwrapper. It behaves as if the wrapper
# is not used
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout_value = proc.communicate()[0]
while proc.returncode is None:
time.wait(60)
stdout_value = proc.communicate()[0]
try:
logdir = os.path.dirname(logfilepath)
apParam.createDirectory(logdir)
file = open(logfilepath,'w')
except:
apDisplay.printError('Log file can not be created, process did not run.')
file.write(stdout_value)
file.close()
if proc.returncode > 0:
pieces = cmd.split(' ')
apDisplay.printWarning('FileScript %s had an error. Please check its log file: \n%s' % (pieces[0].upper(),logfilepath))
else:
apDisplay.printMsg('FileScript ran successfully')
apDisplay.printMsg('------------------------------------------------')
return proc.returncode
def makeDendrogram(numfactors=1, corandata="coran/corandata", dataext=".spi"):
rundir = "cluster"
apParam.createDirectory(rundir)
### make list of factors
factorstr = ""
for fact in range(1,numfactors+1):
factorstr += str(fact)+","
factorstr = factorstr[:-1]
### do hierarchical clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
mySpider.toSpider(
"CL HC",
corandata+"_IMC", # path to coran data
factorstr, # factor string
)
## weight for each factor
for fact in range(numfactors):
mySpider.toSpiderQuiet("1.0")
mySpider.toSpider(
"5", #use Ward's method
"T", "5.1", rundir+"/dendrogram.ps", #dendrogram image file
"Y", rundir+"/dendrogramdoc", #dendrogram doc file
)
mySpider.close()
apImage.convertPostscriptToPng("cluster/dendrogram.ps", "dendrogram.png")
def start(self):
paths = self.getPaths()
for oldpath in paths:
### good path
if oldpath.startswith(self.params['basedir']):
apDisplay.printMsg("Skipping: %s"%(oldpath))
continue
apDisplay.printMsg("Processing: %s"%(oldpath))
### make sure old path exists
if not os.path.isdir(oldpath):
apDisplay.printError("Major error missing directory: %s"%(oldpath))
### get new path
newpath = self.changePath(oldpath)
apParam.createDirectory(os.path.dirname(newpath))
### copy files
shutil.copytree(oldpath, newpath)
### confirm copy
oldsize = self.getPathSize(oldpath)
newsize = self.getPathSize(newpath)
if abs(oldsize - newsize) > oldsize*0.001:
### bigger than 0.1% error
shutil.rmtree(newpath)
apDisplay.printError("Different path sizes: %d versus %d for %s"%(oldsize, newsize, newpath))
apDisplay.printMsg("Good path copy: %d versus %d"%(oldsize,newsize))
### update database
self.updatePath(oldpath, newpath)
### delete old files
shutil.rmtree(oldpath)
def preLoopFunctions(self):
self.data = {}
#if self.params['sessionname'] is not None:
# self.processAndSaveAllImages()
self.params['pickdatadir'] = os.path.join(self.params['rundir'],
"outfiles")
apParam.createDirectory(self.params['pickdatadir'], warning=False)
def breakupStackIntoSingleFiles(stackfile,
partdir="partfiles",
numpart=None,
filetype="spider"):
"""
takes the stack file and creates single spider files ready for processing
"""
apDisplay.printColor(
"Breaking up spider stack into single files, this can take a while",
"cyan")
apParam.createDirectory(partdir)
partdocfile = "partlist.sel"
### setup
breaker = breakUpStack()
if numpart is not None:
breaker.numpart = numpart
breaker.rootdir = partdir
breaker.filetype = filetype
breaker.partdocfile = partdocfile
### make particle files
breaker.start(stackfile)
return partdocfile
def makeStackInFormatFromDefault(stackdata, format="eman"):
"""
Function to create the stack in the requested format
Current default is Imagic format used in EMAN
"""
if format not in formats:
apDisplay.printError("Unknown stack format %s requested" % format)
stackroot, instackdir, outstackdir = getFormattedStackInfo(stackdata, format)
instackpath = os.path.join(instackdir, stackroot + ".hed")
if format == "eman":
return stackdata["path"]["path"]
apParam.createDirectory(outstackdir)
rundir = os.getcwd()
os.chdir(outstackdir)
if format == "spider":
outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0])
emancmd = "proc2d %s %s spidersingle" % (instackpath, outstackpath)
apEMAN.executeEmanCmd(emancmd, showcmd=True)
if format == "frealign":
outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0])
emancmd = "proc2d %s %s invert" % (instackpath, outstackpath)
apEMAN.executeEmanCmd(emancmd, showcmd=True)
if format == "xmipp":
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(instackpath)
os.chdir(rundir)
return outstackdir
def processStack(self, stackarray): self.dirnum += 1 self.partdir = os.path.join(self.rootdir, "%03d"%(self.dirnum)) apParam.createDirectory(self.partdir, warning=False) for partarray in stackarray: self.processParticle(partarray) self.index += 1 #you must have this line in your loop
def ClCla(alignedstack, numpart=None, numclasses=40,
factorlist=range(1,5), corandata="coran/corandata", dataext=".spi"):
"""
this doesn't work
"""
if alignedstack[-4:] == dataext:
alignedstack = alignedstack[:-4]
rundir = "cluster"
classavg = rundir+"/"+("classavgstack%03d" % numclasses)
classvar = rundir+"/"+("classvarstack%03d" % numclasses)
apParam.createDirectory(rundir)
for i in range(numclasses):
apFile.removeFile(rundir+("/classdoc%04d" % (i+1))+dataext)
apFile.removeFile(rundir+"/clusterdoc"+dataext)
factorstr, factorkey = operations.intListToString(factorlist)
### do hierarchical clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider(
"CL CLA",
corandata, # path to coran data
rundir+"/clusterdoc", #clusterdoc file
factorstr, #factor numbers
"5,8",
"4",
"2", # minimum number of particles per class
"Y", rundir+"/dendrogram.ps",
"Y", rundir+"/dendrogramdoc",
)
mySpider.close()
def createTemplateStack(self):
"""
convert spider template stack
"""
templatestack = os.path.join(self.params['rundir'],
"templatestack.spi")
apFile.removeFile(templatestack, warn=True)
### hack to use standard filtering library
templateparams = {}
templateparams['apix'] = self.stack['apix']
templateparams['rundir'] = os.path.join(self.params['rundir'],
"templates")
templateparams['templateIds'] = self.templatelist
templateparams['bin'] = self.params['bin']
templateparams['lowpass'] = self.params['lowpass']
templateparams['median'] = None
templateparams['pixlimit'] = None
print 'Converting reference templates:\n', templateparams
apParam.createDirectory(
os.path.join(self.params['rundir'], "templates"))
filelist = apTemplate.getTemplates(templateparams)
localclip = self.clipsize / self.params['bin']
for mrcfile in filelist:
emancmd = ("proc2d templates/" + mrcfile + " " + templatestack +
" clip=" + str(localclip) + "," + str(localclip) +
" edgenorm spiderswap ")
if self.params['inverttemplates'] is True:
emancmd += " invert "
apEMAN.executeEmanCmd(emancmd, showcmd=False)
return templatestack
def preLoopFunctions(self): apParam.createDirectory(os.path.join(self.params['rundir'], "masks"),warning=False) if self.params['sessionname'] is not None: self.processAndSaveAllImages() self.app = MaskApp() self.app.appionloop = self self.threadJpeg = True
def sortFolder(self):
numsort = 0
apDisplay.printMsg("Sorting files into clean folders")
### move files for all particle iterations
files = []
for i in range(self.lastiter+1):
iterdir = "iter%03d"%(i)
apParam.createDirectory(iterdir, warning=False)
wildcard = "part*_it*%03d_*.*"%(i)
files.extend(glob.glob(wildcard))
wildcard = "part*_it*%03d.*"%(i)
files.extend(glob.glob(wildcard))
for filename in files:
if os.path.isfile(filename):
numsort += 1
shutil.move(filename,iterdir)
if numsort < 3:
apDisplay.printWarning("Problem in iteration file sorting, are they already sorted?")
apDisplay.printMsg("Sorted "+str(numsort)+" iteration files")
### move files for all reference iterations
refsort = 0
refdir = "refalign"
apParam.createDirectory(refdir, warning=False)
wildcard = "ref*_it*.*"
files = glob.glob(wildcard)
for filename in files:
refsort += 1
shutil.move(filename, refdir)
#if refsort < 5:
# apDisplay.printError("Problem in reference file sorting")
apDisplay.printMsg("Sorted "+str(refsort)+" reference files")
return
def preLoopFunctions(self): self.powerspecdir = os.path.join(self.params['rundir'], "opimages") apParam.createDirectory(self.powerspecdir, warning=False) self.ctfrun = None self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) return
def peakTreeToPikFile(peaktree, imgname, tmpl, rundir="."):
outpath = os.path.join(rundir, "pikfiles")
apParam.createDirectory(outpath, warning=False)
outfile = os.path.join(outpath, imgname + "." + str(tmpl) + ".pik")
apFile.removeFile(outfile, warn=True)
#WRITE PIK FILE
f = open(outfile, 'w')
f.write(
"#filename x y mean stdev corr_coeff peak_size templ_num angle moment diam\n"
)
for peakdict in peaktree:
row = peakdict['ycoord']
col = peakdict['xcoord']
if 'corrcoeff' in peakdict:
rho = peakdict['corrcoeff']
else:
rho = 1.0
size = peakdict['peakarea']
mean_str = "%.4f" % peakdict['correlation']
std_str = "%.4f" % peakdict['peakstddev']
mom_str = "%.4f" % peakdict['peakmoment']
if peakdict['diameter'] is not None:
diam_str = "%.2f" % peakdict['diameter']
else:
diam_str = "0"
if 'template' in peakdict:
tmplnum = peakdict['template']
else:
tmplnum = tmpl
#filename x y mean stdev corr_coeff peak_size templ_num angle moment
out = imgname+".mrc "+str(int(col))+" "+str(int(row))+ \
" "+mean_str+" "+std_str+" "+str(rho)+" "+str(int(size))+ \
" "+str(tmplnum)+" 0 "+mom_str+" "+diam_str
f.write(str(out) + "\n")
f.close()
def writeMultipleFreeHandTestShells(self, constant_inputlines, nproc):
scripts = []
files = []
last_particle = apStack.getNumberStackParticlesFromId(self.params["stackid"], msg=True)
last_particle = 82
# This is integer division and will return integer
stepsize = int(math.ceil(float(last_particle) / nproc))
for proc in range(nproc):
proc_start_particle = stepsize * proc + 1
proc_end_particle = min(stepsize * (proc + 1), last_particle)
proc_inputlines = list(constant_inputlines)
proc_inputlines.insert(8, "%d, %d" % (proc_start_particle, proc_end_particle))
proc_inputlines.extend(self.createTiltedCtfLines(proc_start_particle, proc_end_particle))
procpath = os.path.join(self.params["rundir"], "proc%03d" % (proc))
apParam.createDirectory(procpath, warning=False)
lines_before_input = [
"#!/bin/csh",
"# Proc %03d, Particles %d - %d" % (proc, proc_start_particle, proc_end_particle),
"rm -rf %s" % procpath,
"mkdir %s" % procpath,
"cd %s" % procpath,
"",
"",
"### START FREALIGN ###",
"/home/acheng/bin/fastfreehand_v1_01.exe << EOF > freehand.proc%03d.out" % proc,
]
lines_after_input = ["EOF", "", "### END FREEHAND", 'echo "END FREEHAND"']
alllines = lines_before_input + proc_inputlines + lines_after_input
procfile = "freehand.proc%03d.csh" % (proc)
f = open(procfile, "w")
f.writelines(map((lambda x: x + "\n"), alllines))
f.close()
os.chmod(procfile, 0755)
scripts.append(procfile)
return scripts
def makeDendrogram(numfactors=1, corandata="coran/corandata", dataext=".spi"):
rundir = "cluster"
apParam.createDirectory(rundir)
### make list of factors
factorstr = ""
for fact in range(1, numfactors + 1):
factorstr += str(fact) + ","
factorstr = factorstr[:-1]
### do hierarchical clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
mySpider.toSpider(
"CL HC",
corandata + "_IMC", # path to coran data
factorstr, # factor string
)
## weight for each factor
for fact in range(numfactors):
mySpider.toSpiderQuiet("1.0")
mySpider.toSpider(
"5", #use Ward's method
"T",
"5.1",
rundir + "/dendrogram.ps", #dendrogram image file
"Y",
rundir + "/dendrogramdoc", #dendrogram doc file
)
mySpider.close()
apImage.convertPostscriptToPng("cluster/dendrogram.ps", "dendrogram.png")
def createSubFolders(partdir, numpart, filesperdir): i = 0 dirnum = 0 while i < numpart: dirnum += 1 apParam.createDirectory(os.path.join(partdir, str(dirnum)), warning=False) i += filesperdir return dirnum
def preLoopFunctions(self):
apParam.createDirectory(os.path.join(self.params['rundir'], "masks"),
warning=False)
if self.params['sessionname'] is not None:
self.processAndSaveAllImages()
self.app = MaskApp()
self.app.appionloop = self
self.threadJpeg = True
def preLoopFunctions(self):
self.ctfrun = None
self.powerspecdir = os.path.join(self.params['rundir'], "opimages")
apParam.createDirectory(self.powerspecdir, warning=False)
self.logdir = os.path.join(self.params['rundir'], "logfiles")
apParam.createDirectory(self.logdir, warning=False)
self.ctfprgmexe = self.getCtfProgPath()
return
def preLoopFunctions(self): self.ctfrun = None self.powerspecdir = os.path.join(self.params['rundir'], "opimages") apParam.createDirectory(self.powerspecdir, warning=False) self.logdir = os.path.join(self.params['rundir'], "logfiles") apParam.createDirectory(self.logdir, warning=False) self.ctfprgmexe = self.getCtfProgPath() return
def preLoopFunctions(self): self.params['pickdatadir'] = os.path.join(self.params['rundir'],"pickdata") apParam.createDirectory(self.params['pickdatadir'], warning=False) if self.params['sessionname'] is not None: self.processAndSaveAllImages() self.app = ApTiltPicker.PickerApp(mode='loop') self.app.appionloop = self self.theta = 0.0
def createFactorMap(f1, f2, rundir, dataext):
### 3. factor plot
apParam.createDirectory(rundir + "/factors", warning=False)
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
factorfile = rundir + "/factors/factorps" + ("%02d-%02d" % (f1, f2))
mySpider.toSpiderQuiet(
"CA SM",
"I",
rundir + "/corandata", #coran prefix
"0",
str(f1) + "," + str(f2), #factors to plot
"S",
"+",
"Y",
"5",
"0",
factorfile,
"\n\n\n\n",
"\n\n\n\n",
"\n", #9 extra steps, use defaults
)
time.sleep(2)
mySpider.close()
# hack to get postscript converted to png, require ImageMagick
apImage.convertPostscriptToPng(factorfile + ".ps",
factorfile + ".png",
size=200)
apFile.removeFile(factorfile + ".ps")
### 4. factor plot visualization
"""
### this is broken in SPIDER 13.0
mySpider = spyder.SpiderSession(dataext=dataext, logo=False)
mySpider.toSpider(
"SD C", #create coordinate file
rundir+"/corandata", #coran prefix
str(f1)+","+str(f2), #factors to plot
rundir+"/sdcdoc"+("%02d%02d" % (f1,f2)),
)
visimg = rundir+"/visimg"+("%02d%02d" % (f1,f2))
mySpider.toSpider(
"CA VIS", #visualization
"(1024,1024)",
rundir+"/sdcdoc"+("%02d%02d" % (f1,f2)), #input doc from 'sd c'
rundir+"/visdoc"+("%02d%02d" % (f1,f2)), #output doc
"alignedstack@00001", # image in series ???
"(12,12)", #num of rows, cols
"5.0", #stdev range
"(5.0,5.0)", #upper, lower thresh
visimg, #output image
"1,"+str(numpart),
"1,2",
)
mySpider.close()
emancmd = ("proc2d "+visimg+dataext+" "+visimg+".png ")
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
"""
return
def commitToDatabase(self, imgdata):
# if a kept mask was created in a previous mask run and the
# assess flag was used (basically combining the 2 runs) there is
# nothing new to commit.
if self.useAcceptedMask: return
sessiondata = imgdata['session']
rundir = self.params['rundir']
maskname = self.params['runname']
assessname = self.params['assessname']
bin = self.params['bin']
maskdir = os.path.join(rundir, "masks")
maskrundata, maskparamsdata = apMask.getMaskParamsByRunName(
maskname, sessiondata)
if not maskrundata:
apMask.insertManualMaskRun(sessiondata, rundir, maskname, bin)
maskrundata, maskparamsdata = apMask.getMaskParamsByRunName(
maskname, sessiondata)
try:
apParam.createDirectory(maskdir)
except:
apDisplay.printWarning('can not create mask directory')
massessrundata, exist = apMask.insertMaskAssessmentRun(
sessiondata, maskrundata, assessname)
mask = self.maskimg
maskfilename = imgdata['filename'] + '_mask.png'
image = self.image
labeled_regions, clabels = nd.label(mask)
testlog = [False, 0, ""]
infos = {}
infos, testlog = apCrud.getLabeledInfo(image, mask, labeled_regions,
range(1, clabels + 1), False,
infos, testlog)
offset = 1
for l1 in range(0, len(infos)):
l = l1 + offset
info = infos[l]
info.append(l)
regiondata = apMask.insertMaskRegion(maskrundata, imgdata, info)
print "Inserting mask region in database"
# PIL alpha channel read does not work
#apImage.arrayMaskToPngAlpha(mask, os.path.join(maskdir,maskfilename))
apImage.arrayMaskToPng(mask, os.path.join(maskdir, maskfilename))
allregiondata = apMask.getMaskRegions(maskrundata, imgdata)
for regiondata in allregiondata:
apMask.insertMaskAssessment(massessrundata, regiondata, 1)
print "Inserting mask assessment in database."
if self.assess != self.assessold and self.assess is not None:
#imageaccessor run is always named run1
apDatabase.insertImgAssessmentStatus(imgdata, 'run1', self.assess)
return
def createSubFolders(partdir, numpart, filesperdir):
i = 0
dirnum = 0
while i < numpart:
dirnum += 1
apParam.createDirectory(os.path.join(partdir, str(dirnum)),
warning=False)
i += filesperdir
return dirnum
def createIterationJobRefineFiles(self, oldprefix, newprefix, stackFile):
"""
Create multiple job files for frealign reconstruction
using the mpiexec command
"""
### create individual mpi scripts
particlePerProcess = float(
self.numpart) / self.params['totalprocs'] - 1
apDisplay.printColor(
"Using approx %.1f particles per process file, for a total of %d processes"
% (particlePerProcess, self.params['totalprocs']), "purple")
lastPart = 0
procjobfiles = []
for i in range(self.params['totalprocs']):
procNum = i + 1
inputVolFile = "../threed.%s.mrc" % (oldprefix)
firstPart = lastPart + 1
lastPart = firstPart + particlePerProcess
if lastPart > self.numpart - 2:
lastPart = self.numpart
inputParamFile = "../params.%s.par" % (oldprefix)
apParam.createDirectory(newprefix, warning=False)
jobFile = "%s/refine.%s.proc%03d.sh" % (newprefix, newprefix,
procNum)
procPrefix = "%s.proc%03d" % (newprefix, procNum)
f = open(jobFile, 'w')
f.write("#!/bin/sh\n\n")
fullpath = os.path.join(self.params['rundir'], newprefix)
f.write('cd %s\n' % fullpath)
f.write("/bin/rm -fv frealign.%s.out\n" % (procPrefix))
f.write("/bin/rm -fv outparams.%s.par\n" % (procPrefix))
f.write("/bin/rm -fv shift.%s.par\n" % (procPrefix))
f.write("mkdir /tmp/frealign\n")
f.write("rsync -vrtPhL %s /tmp/frealign/%s\n" %
(inputVolFile, os.path.basename(inputVolFile)))
f.write("rsync -vrtPhL %s /tmp/frealign/%s\n" %
(stackFile, os.path.basename(stackFile)))
f.close()
#partDiff = math.floor(lastPart) - math.floor(firstPart)
#print "proc %d: %.1f->%.1f (%d)"%(procNum, firstPart, lastPart, partDiff)
self.appendFrealignJobFile(jobFile,
inputParamFile,
inputVolFile,
stackFile,
math.floor(firstPart),
math.floor(lastPart),
procPrefix,
recon=False)
return
def createMultipleJobs(self, iternum):
"""
Create multiple job files for frealign reconstruction
using the mpiexec command
"""
### create script that will launch all mpi scripts
iterdir = "iter%03d" % (iternum)
iterjobfile = 'frealign.iter%03d.run.sh' % (iternum)
#frscript = os.path.join(workdir,'frealign.$PBS_VNODENUM.sh')
#fr.write("sh "+frscript+"\n")
### create individual mpi scripts
partperjob = math.floor(self.params['last'] / self.params['nproc'])
#apDisplay.printMsg("%d particles per processor"%(partperjob))
r = self.params['last'] % self.params['nproc']
lastp = 0
procjobfiles = []
for n in range(self.params['nproc']):
firstp = lastp + 1
lastp = firstp + partperjob - 1
if r > 0:
lastp += 1
r -= 1
procdir = "iter%03d/proc%03d" % (iternum, n + 1)
#print procdir
apParam.createDirectory("iter%03d" % (iternum), warning=False)
procjobfile = "iter%03d/frealign.iter%03d.proc%03d.sh" % (
iternum, iternum, n + 1)
### start jobfile
f = open(procjobfile, "w")
f.write("#!/bin/sh\n")
f.write("\n")
f.write("### Job #%03d, Particles %6d - %6d\n" %
(n + 1, firstp, lastp))
f.write('mkdir -p %s\n' % procdir)
f.write('cd %s\n' % procdir)
time.sleep(0.05)
f.close()
### add frealign code
self.currentvol = "../../" + os.path.basename(self.currentvol)
self.currentparam = "../../" + os.path.basename(self.currentparam)
logfile = "frealign.proc%03d.out" % (n + 1)
self.appendFrealignJobFile(procjobfile,
first=firstp,
last=lastp,
recon=False,
logfile=logfile)
### append to list
procjobfiles.append(procjobfile)
return procjobfiles
def start(self):
commit = self.params['commit']
description = self.params['description']
processdir = self.params['fulltomodir']
runname = self.params['runname']
offsetz = self.params['offsetz']
subbin = self.params['bin']
invert = self.params['invert']
fulltomodata = apTomo.getFullTomoData(self.params['fulltomoId'])
#subvolume making
if (self.params['selexonId'] is not None or self.params['stackId']) and fulltomodata is not None:
sessiondata = fulltomodata['session']
seriesname = fulltomodata['name'].rstrip('_full')
fullbin = fulltomodata['bin']
if not fullbin:
apDisplay.printWarning("no binning in full tomogram, something is wrong, use alignment bin for now")
fullbin = fulltomodata['aligner']['alignrun']['bin']
fulltomopath = os.path.join(fulltomodata['reconrun']['path']['path'], seriesname+"_full.rec")
fulltomoheader = mrc.readHeaderFromFile(fulltomopath)
fulltomoshape = fulltomoheader['shape']
if self.params['sizez'] > fulltomoshape[1]*fullbin :
self.params['sizez'] = fulltomoshape[1]*fullbin
subrunname = self.params['subrunname']
volumeindex = apTomo.getLastVolumeIndex(fulltomodata) + 1
dimension = {'x':int(self.params['sizex']),'y':int(self.params['sizey']),'z':int(self.params['sizez'])}
zprojimagedata = fulltomodata['zprojection']
apDisplay.printMsg("getting pixelsize")
pixelsize = apTomo.getTomoPixelSize(zprojimagedata) * fullbin * subbin
gtransform = [1,0,0,1,0,0]
if self.params['selexonId']:
particles = apParticle.getParticles(zprojimagedata, self.params['selexonId'])
if self.params['stackId']:
particles,stackparticles = apStack.getImageParticles(zprojimagedata, self.params['stackId'])
stackdata = apStack.getOnlyStackData(self.params['stackId'])
for p, particle in enumerate(particles):
print particle['xcoord'],particle['ycoord'],fullbin
center = apTomo.transformParticleCenter(particle,fullbin,gtransform)
size = (dimension['x']/fullbin,dimension['y']/fullbin,dimension['z']/fullbin)
volumename = 'volume%d'% (volumeindex,)
volumedir = os.path.join(processdir,subrunname+'/',volumename+'/')
apParam.createDirectory(volumedir)
apImod.trimVolume(processdir, subrunname,seriesname,volumename,center,offsetz,size,True)
long_volumename = seriesname+'_'+volumename
subvolumepath = os.path.join(processdir, runname+"/",volumename+"/",long_volumename+".rec")
if subbin > 1 or invert:
apTomo.modifyVolume(subvolumepath,subbin,invert)
if commit:
subtomorundata = apTomo.insertSubTomoRun(sessiondata,
self.params['selexonId'],self.params['stackId'],subrunname,invert,subbin)
subtomodata = apTomo.insertSubTomogram(fulltomodata,subtomorundata,
particle,offsetz,dimension,
volumedir,long_volumename,volumeindex,pixelsize,
description)
apTomo.makeMovie(subvolumepath)
apTomo.makeProjection(subvolumepath)
volumeindex += 1
def start(self):
commit = self.params['commit']
description = self.params['description']
processdir = self.params['fulltomodir']
runname = self.params['runname']
offsetz = self.params['offsetz']
subbin = self.params['bin']
invert = self.params['invert']
fulltomodata = apTomo.getFullTomoData(self.params['fulltomoId'])
#subvolume making
if (self.params['selexonId'] is not None or self.params['stackId']) and fulltomodata is not None:
sessiondata = fulltomodata['session']
seriesname = fulltomodata['name'].rstrip('_full')
fullbin = fulltomodata['bin']
if not fullbin:
apDisplay.printWarning("no binning in full tomogram, something is wrong, use alignment bin for now")
fullbin = fulltomodata['aligner']['alignrun']['bin']
fulltomopath = os.path.join(fulltomodata['reconrun']['path']['path'], seriesname+"_full.rec")
fulltomoheader = mrc.readHeaderFromFile(fulltomopath)
fulltomoshape = fulltomoheader['shape']
if self.params['sizez'] > fulltomoshape[1]*fullbin :
self.params['sizez'] = fulltomoshape[1]*fullbin
subrunname = self.params['subrunname']
volumeindex = apTomo.getLastVolumeIndex(fulltomodata) + 1
dimension = {'x':int(self.params['sizex']),'y':int(self.params['sizey']),'z':int(self.params['sizez'])}
zprojimagedata = fulltomodata['zprojection']
apDisplay.printMsg("getting pixelsize")
pixelsize = apTomo.getTomoPixelSize(zprojimagedata) * fullbin * subbin
gtransform = [1,0,0,1,0,0]
if self.params['selexonId']:
particles = apParticle.getParticles(zprojimagedata, self.params['selexonId'])
if self.params['stackId']:
particles,stackparticles = apStack.getImageParticles(zprojimagedata, self.params['stackId'])
stackdata = apStack.getOnlyStackData(self.params['stackId'])
for p, particle in enumerate(particles):
print particle['xcoord'],particle['ycoord'],fullbin
center = apTomo.transformParticleCenter(particle,fullbin,gtransform)
size = (dimension['x']/fullbin,dimension['y']/fullbin,dimension['z']/fullbin)
volumename = 'volume%d'% (volumeindex,)
volumedir = os.path.join(processdir,subrunname+'/',volumename+'/')
apParam.createDirectory(volumedir)
apImod.trimVolume(processdir, subrunname,seriesname,volumename,center,offsetz,size,True)
long_volumename = seriesname+'_'+volumename
subvolumepath = os.path.join(processdir, runname+"/",volumename+"/",long_volumename+".rec")
if subbin > 1 or invert:
apTomo.modifyVolume(subvolumepath,subbin,invert)
if commit:
subtomorundata = apTomo.insertSubTomoRun(sessiondata,
self.params['selexonId'],self.params['stackId'],subrunname,invert,subbin)
subtomodata = apTomo.insertSubTomogram(fulltomodata,subtomorundata,
particle,offsetz,dimension,
volumedir,long_volumename,volumeindex,pixelsize,
description)
apTomo.makeMovie(subvolumepath)
apTomo.makeProjection(subvolumepath)
volumeindex += 1
def hierarchCluster(alignedstack, numpart=None, numclasses=40, timestamp=None, factorlist=range(1,5), corandata="coran/corandata", dataext=".spi"): rundir = "cluster" apParam.createDirectory(rundir) ### step 1: use coran data to create hierarchy dendrogramfile = hierarchClusterProcess(numpart, factorlist, corandata, rundir, dataext) ### step 2: asssign particles to groups based on hierarchy classavg,classvar = hierarchClusterClassify(alignedstack, dendrogramfile, numclasses, timestamp, rundir, dataext) return classavg,classvar
def __init__(self,rundir):
self.rundir = rundir
self.ksvd_param_line = None
self.stack_param_line = None
self.matlabpaths = []
if os.environ.has_key('MATLAB_KSVD_TOOL_PATH') and os.environ.has_key('MATLAB_DENOISE_PATH'):
self.setHelperPaths(os.environ['MATLAB_KSVD_TOOL_PATH'],os.environ['MATLAB_DENOISE_PATH'])
apParam.createDirectory(os.path.join(rundir,'results','mrc'))
apParam.createDirectory(os.path.join(rundir,'results','dict'))
def setSubVolumePath(self,volumeindex): processdir = self.params['fulltomodir'] subrunname = self.params['subrunname'] volumename = 'volume%d'% (volumeindex,) volumedir = os.path.join(processdir,subrunname+'/',volumename+'/') apParam.createDirectory(volumedir) long_volumename = self.seriesname+'_'+volumename subvolumepath = os.path.join(processdir, subrunname+"/",volumename+"/",long_volumename+".rec") self.subvolumepath = subvolumepath return volumename
def hierarchCluster(alignedstack, numpart=None, numclasses=40, timestamp=None,
factorlist=range(1,5), corandata="coran/corandata", dataext=".spi"):
rundir = "cluster"
apParam.createDirectory(rundir)
### step 1: use coran data to create hierarchy
dendrogramfile = hierarchClusterProcess(numpart, factorlist, corandata, rundir, dataext)
### step 2: asssign particles to groups based on hierarchy
classavg,classvar = hierarchClusterClassify(alignedstack, dendrogramfile, numclasses, timestamp, rundir, dataext)
return classavg,classvar
def processImage(self, imgdata):
#image = self.getImage(imgdata, self.params['bin'])
self.image = imgdata['image']
# Check the image shape
imgshape = numpy.asarray(imgdata['image'].shape)
apDisplay.printMsg("MRC Image Shape prior to processing:")
print imgshape
imagepath = os.path.join(imgdata['session']['image path'],
imgdata['filename'] + ".mrc")
# Convert the MRC image to a jpg for find_mask.py
apDisplay.printMsg("Converting mrc image to jpg.")
jpg_dir = os.path.join(self.params['rundir'], "jpgs")
jpg_image = os.path.join(jpg_dir, imgdata['filename'] + ".jpg")
apParam.createDirectory(jpg_dir, warning=False)
pyami.numpil.write(self.image, jpg_image)
if (self.params['test']):
self.outfile = os.path.join(self.params['rundir'], "tests",
imgdata['filename'] + "_mask.jpg")
else:
self.outfile = os.path.join(self.params['rundir'], "masks",
imgdata['filename'] + "_mask.jpg")
downsample = str(self.params['downsample'])
compsizethresh = str(self.params['compsizethresh'])
adapthresh = str(self.params['adapthresh'])
dilation = str(self.params['dilation'])
erosion = str(self.params['erosion'])
blur = str(self.params['blur'])
options = " --downsample=" + downsample + " --compsizethresh=" + compsizethresh + " --adapthresh=" + adapthresh + " --blur=" + blur + " --dilation=" + dilation + " --erosion=" + erosion
commandline = ("source " + self.activatepath +
"; python `which find_mask.py` --ifile=" + jpg_image +
" --ofile=" + self.outfile + options + "\n")
# Test with test image
#commandline = ( "source /opt/em_hole_finder/env/bin/activate; python /opt/em_hole_finder/find_mask_amber.py \n" )
### run command
apDisplay.printMsg("running em hole finder " + time.asctime())
apDisplay.printColor(commandline, "purple")
if True:
proc = subprocess.Popen(commandline, shell=True)
else:
outf = open("automasker.out", "a")
errf = open("automasker.err", "a")
proc = subprocess.Popen(commandline,
shell=True,
stderr=errf,
stdout=outf)
proc.wait()
def createTiltedPeakJpeg(imgdata1, imgdata2, peaktree1, peaktree2, params, procimg1=None, procimg2=None):
if 'templatelist' in params:
count = len(params['templatelist'])
else: count = 1
bin = int(params["bin"])
diam = float(params["diam"])
apix = float(params["apix"])
binpixrad = diam/apix/2.0/float(bin)
imgname1 = imgdata1['filename']
imgname2 = imgdata2['filename']
jpegdir = os.path.join(params['rundir'],"jpgs")
apParam.createDirectory(jpegdir, warning=False)
if procimg1 is not None:
imgarray1 = procimg1
else:
imgarray1 = apImage.preProcessImage(imgdata1['image'], bin=bin, planeReg=False, params=params)
if procimg2 is not None:
imgarray2 = procimg2
else:
imgarray2 = apImage.preProcessImage(imgdata2['image'], bin=bin, planeReg=False, params=params)
imgarray = numpy.hstack((imgarray1,imgarray2))
image = apImage.arrayToImage(imgarray)
image = image.convert("RGB")
image2 = image.copy()
draw = ImageDraw.Draw(image2)
#import pprint
if len(peaktree1) > 0:
#pprint.pprint(peaktree1)
drawPeaks(peaktree1, draw, bin, binpixrad)
if len(peaktree2) > 0:
peaktree2adj = []
for peakdict in peaktree2:
peakdict2adj = {}
#pprint.pprint(peakdict)
peakdict2adj['xcoord'] = peakdict['xcoord'] + imgdata1['image'].shape[1]
peakdict2adj['ycoord'] = peakdict['ycoord']
peakdict2adj['peakarea'] = 1
peakdict2adj['tmplnum'] = 2
peaktree2adj.append(peakdict2adj.copy())
#pprint.pprint(peaktree2adj)
drawPeaks(peaktree2adj, draw, bin, binpixrad)
image = Image.blend(image, image2, 0.9)
outfile1 = os.path.join(jpegdir, imgname1+".prtl.jpg")
apDisplay.printMsg("writing peak JPEG: "+outfile1)
image.save(outfile1, "JPEG", quality=95)
outfile2 = os.path.join(jpegdir, imgname2+".prtl.jpg")
apDisplay.printMsg("writing peak JPEG: "+outfile2)
image.save(outfile2, "JPEG", quality=95)
return
def createTiltedPeakJpeg(imgdata1, imgdata2, peaktree1, peaktree2, params, procimg1=None, procimg2=None):
if 'templatelist' in params:
count = len(params['templatelist'])
else: count = 1
bin = int(params["bin"])
diam = float(params["diam"])
apix = float(params["apix"])
binpixrad = diam/apix/2.0/float(bin)
imgname1 = imgdata1['filename']
imgname2 = imgdata2['filename']
jpegdir = os.path.join(params['rundir'],"jpgs")
apParam.createDirectory(jpegdir, warning=False)
if procimg1 is not None:
imgarray1 = procimg1
else:
imgarray1 = apImage.preProcessImage(imgdata1['image'], bin=bin, planeReg=False, params=params)
if procimg2 is not None:
imgarray2 = procimg2
else:
imgarray2 = apImage.preProcessImage(imgdata2['image'], bin=bin, planeReg=False, params=params)
imgarray = numpy.hstack((imgarray1,imgarray2))
image = apImage.arrayToImage(imgarray)
image = image.convert("RGB")
image2 = image.copy()
draw = ImageDraw.Draw(image2)
#import pprint
if len(peaktree1) > 0:
#pprint.pprint(peaktree1)
drawPeaks(peaktree1, draw, bin, binpixrad)
if len(peaktree2) > 0:
peaktree2adj = []
for peakdict in peaktree2:
peakdict2adj = {}
#pprint.pprint(peakdict)
peakdict2adj['xcoord'] = peakdict['xcoord'] + imgdata1['image'].shape[1]
peakdict2adj['ycoord'] = peakdict['ycoord']
peakdict2adj['peakarea'] = 1
peakdict2adj['tmplnum'] = 2
peaktree2adj.append(peakdict2adj.copy())
#pprint.pprint(peaktree2adj)
drawPeaks(peaktree2adj, draw, bin, binpixrad)
image = Image.blend(image, image2, 0.9)
outfile1 = os.path.join(jpegdir, imgname1+".prtl.jpg")
apDisplay.printMsg("writing peak JPEG: "+outfile1)
image.save(outfile1, "JPEG", quality=95)
outfile2 = os.path.join(jpegdir, imgname2+".prtl.jpg")
apDisplay.printMsg("writing peak JPEG: "+outfile2)
image.save(outfile2, "JPEG", quality=95)
return
def setProtomoDir(rootdir, cycle=0):
print "Setting up directories"
rawdir = os.path.join(rootdir, 'raw')
aligndir = os.path.join(rootdir, 'align')
outdir = os.path.join(rootdir, 'out')
cleandir = os.path.join(rootdir, 'clean')
for dir in (rawdir, aligndir, outdir, cleandir):
if os.path.exists(dir) and cycle == 0:
apDisplay.printWarning('removing existing directory %s' % (dir, ))
shutil.rmtree(dir)
apParam.createDirectory(dir, warning=False)
return aligndir, rawdir
def start(self):
#find stack
stackparticles = apStack.getStackParticlesFromId(
self.params['stackid'])
if self.params['logsplit']:
#stacklist = oldLogSplit(self.params['logstart'], len(stackparticles), self.params['logdivisions'])
stacklist = evenLogSplit(self.params['logstart'],
len(stackparticles))
elif self.params['nptcls']:
stacklist = [self.params['nptcls']]
else:
apDisplay.printError("Please specify nptlcs or logsplit")
oldstackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(oldstackdata['path']['path'],
oldstackdata['name'])
#create run directory
if self.params['rundir'] is None:
path = oldstackdata['path']['path']
path = os.path.split(os.path.abspath(path))[0]
self.params['rundir'] = path
apDisplay.printMsg("Out directory: " + self.params['rundir'])
origdescription = self.params['description']
for stack in stacklist:
self.params['description'] = (
origdescription +
(" ... split %d particles from original stackid=%d" %
(stack, self.params['stackid'])))
workingdir = os.path.join(self.params['rundir'], str(stack))
#check for previously commited stacks
newstack = os.path.join(workingdir, self.params['stackname'])
apStack.checkForPreviousStack(newstack)
#create rundir and change to that directory
apDisplay.printMsg("Run directory: " + workingdir)
apParam.createDirectory(workingdir)
os.chdir(workingdir)
#create random list
lstfile = makeRandomLst(stack, stackparticles, self.params)
#shutil.copy(lstfile, workingdir)
#make new stack
apStack.makeNewStack(oldstack, newstack, lstfile)
#apStack.makeNewStack(lstfile, self.params['stackname'])
#commit new stack
self.params['keepfile'] = os.path.abspath(lstfile)
self.params['rundir'] = os.path.abspath(workingdir)
apStack.commitSubStack(self.params)
def setProtomoDir(rootdir,cycle=0):
print "Setting up directories"
rawdir=os.path.join(rootdir, 'raw')
aligndir=os.path.join(rootdir,'align')
outdir=os.path.join(rootdir,'out')
cleandir=os.path.join(rootdir,'clean')
for dir in (rawdir,aligndir,outdir,cleandir):
if os.path.exists(dir) and cycle==0:
apDisplay.printWarning('removing existing directory %s' % (dir,))
shutil.rmtree(dir)
apParam.createDirectory(dir,warning=False)
return aligndir, rawdir
def __init__(self, rundir):
self.rundir = rundir
self.ksvd_param_line = None
self.stack_param_line = None
self.matlabpaths = []
if os.environ.has_key('MATLAB_KSVD_TOOL_PATH') and os.environ.has_key(
'MATLAB_DENOISE_PATH'):
self.setHelperPaths(os.environ['MATLAB_KSVD_TOOL_PATH'],
os.environ['MATLAB_DENOISE_PATH'])
apParam.createDirectory(os.path.join(rundir, 'results', 'mrc'))
apParam.createDirectory(os.path.join(rundir, 'results', 'dict'))
def commitToDatabase(self,imgdata):
# if a kept mask was created in a previous mask run and the
# assess flag was used (basically combining the 2 runs) there is
# nothing new to commit.
if self.useAcceptedMask: return
sessiondata = imgdata['session']
rundir = self.params['rundir']
maskname = self.params['runname']
assessname = self.params['assessname']
bin = self.params['bin']
maskdir=os.path.join(rundir,"masks")
maskrundata,maskparamsdata = apMask.getMaskParamsByRunName(maskname,sessiondata)
if not maskrundata:
apMask.insertManualMaskRun(sessiondata,rundir,maskname,bin)
maskrundata,maskparamsdata = apMask.getMaskParamsByRunName(maskname,sessiondata)
try:
apParam.createDirectory(maskdir)
except:
apDisplay.printWarning('can not create mask directory')
massessrundata,exist = apMask.insertMaskAssessmentRun(sessiondata,maskrundata,assessname)
mask = self.maskimg
maskfilename = imgdata['filename']+'_mask.png'
image = self.image
labeled_regions,clabels=nd.label(mask)
testlog = [False,0,""]
infos={}
infos,testlog=apCrud.getLabeledInfo(image,mask,labeled_regions,range(1,clabels+1),False,infos,testlog)
offset=1
for l1 in range(0,len(infos)):
l=l1+offset
info=infos[l]
info.append(l)
regiondata= apMask.insertMaskRegion(maskrundata,imgdata,info)
print "Inserting mask region in database"
# PIL alpha channel read does not work
#apImage.arrayMaskToPngAlpha(mask, os.path.join(maskdir,maskfilename))
apImage.arrayMaskToPng(mask, os.path.join(maskdir,maskfilename))
allregiondata = apMask.getMaskRegions(maskrundata,imgdata)
for regiondata in allregiondata:
apMask.insertMaskAssessment(massessrundata,regiondata,1)
print "Inserting mask assessment in database."
if self.assess != self.assessold and self.assess is not None:
#imageaccessor run is always named run1
apDatabase.insertImgAssessmentStatus(imgdata, 'run1', self.assess)
return
def processImage(self,imgdata):
image = self.getImage(imgdata,self.params['bin'])
results=self.function(self.rundata, imgdata, image)
mask = results.pop('mask')
pngpath = os.path.join(self.params['rundir'],"masks")
apParam.createDirectory(pngpath,warning=False)
filepathname = os.path.join(self.params['rundir'],"masks",imgdata['filename']+"_mask.png")
if mask is not None:
# PIL alpha channel read does not work
#apImage.arrayMaskToPngAlpha(mask, filepathname)
apImage.arrayMaskToPng(mask, filepathname)
return results
def preLoopFunctions(self):
self.params['opimage1'] = os.path.join(self.params['rundir'],"opimages1")
self.params['opimage2'] = os.path.join(self.params['rundir'],"opimages2")
self.params['tempdir'] = os.path.join(self.params['rundir'],"tempdir")
apParam.createDirectory(self.params['opimage1'], warning=False)
apParam.createDirectory(self.params['opimage2'], warning=False)
apParam.createDirectory(self.params['tempdir'], warning=False)
for p in ("doubtful", "questionable", "probable", "all"):
path = os.path.join(self.params['opimage1'], p)
apParam.createDirectory(path, warning=False)
path = os.path.join(self.params['opimage2'], p)
apParam.createDirectory(path, warning=False)
def start(self):
#find stack
stackparticles = apStack.getStackParticlesFromId(self.params['stackid'])
if self.params['logsplit']:
#stacklist = oldLogSplit(self.params['logstart'], len(stackparticles), self.params['logdivisions'])
stacklist = evenLogSplit(self.params['logstart'], len(stackparticles))
elif self.params['nptcls']:
stacklist = [self.params['nptcls']]
else:
apDisplay.printError("Please specify nptlcs or logsplit")
oldstackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(oldstackdata['path']['path'], oldstackdata['name'])
#create run directory
if self.params['rundir'] is None:
path = oldstackdata['path']['path']
path = os.path.split(os.path.abspath(path))[0]
self.params['rundir'] = path
apDisplay.printMsg("Out directory: "+self.params['rundir'])
origdescription=self.params['description']
for stack in stacklist:
self.params['description'] = (
origdescription+
(" ... split %d particles from original stackid=%d"
% (stack, self.params['stackid']))
)
workingdir = os.path.join(self.params['rundir'], str(stack))
#check for previously commited stacks
newstack = os.path.join(workingdir ,self.params['stackname'])
apStack.checkForPreviousStack(newstack)
#create rundir and change to that directory
apDisplay.printMsg("Run directory: "+workingdir)
apParam.createDirectory(workingdir)
os.chdir(workingdir)
#create random list
lstfile = makeRandomLst(stack, stackparticles, self.params)
#shutil.copy(lstfile, workingdir)
#make new stack
apStack.makeNewStack(oldstack, newstack, lstfile)
#apStack.makeNewStack(lstfile, self.params['stackname'])
#commit new stack
self.params['keepfile'] = os.path.abspath(lstfile)
self.params['rundir'] = os.path.abspath(workingdir)
apStack.commitSubStack(self.params)
