def convertPostscriptToPng(psfile, pngfile, size=1024):
### better pstopnm pre-step
pstopnmcmd = "pstopnm -xsize=2000 -ysize=2000 -xborder=0 -yborder=0 -portrait " + psfile
proc = subprocess.Popen(pstopnmcmd, shell=True)
proc.wait()
### direct conversion
ppmfile = os.path.splitext(psfile)[0] + "001.ppm"
if os.path.isfile(ppmfile):
imagemagickcmd = ("convert -colorspace Gray -trim -resize " +
str(size) + "x" + str(size) + " " + ppmfile + " " +
pngfile)
else:
ppmfile = psfile + "001.ppm"
if os.path.isfile(ppmfile):
imagemagickcmd = ("convert -colorspace Gray -trim -resize " +
str(size) + "x" + str(size) + " " + ppmfile +
" " + pngfile)
else:
imagemagickcmd = ("convert -colorspace Gray -trim -resize " +
str(size) + "x" + str(size) + " " + psfile +
" " + pngfile)
proc = subprocess.Popen(imagemagickcmd, shell=True)
proc.wait()
if os.path.isfile(ppmfile):
apFile.removeFile(ppmfile)
if not os.path.isfile(pngfile):
apDisplay.printWarning("Postscript image conversion failed")
def createTemplateStack(self):
"""
takes the spider file and creates an average template of all particles
"""
templatestack = os.path.join(self.params['rundir'], "templatestack00.spi")
apFile.removeFile(templatestack, warn=True)
### hack to use standard filtering library
templateparams = {}
templateparams['apix'] = self.stack['apix']
templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
templateparams['templateIds'] = self.templatelist
templateparams['bin'] = self.params['bin']
templateparams['lowpass'] = self.params['lowpass']
templateparams['median'] = None
templateparams['pixlimit'] = None
print templateparams
apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
filelist = apTemplate.getTemplates(templateparams)
for mrcfile in filelist:
emancmd = ("proc2d templates/"+mrcfile+" "+templatestack
+" clip="+str(self.boxsize)+","+str(self.boxsize)
+" spider ")
if self.params['inverttemplates'] is True:
emancmd += " invert "
apEMAN.executeEmanCmd(emancmd, showcmd=False)
return templatestack
def xmippNormStack(self, inStackPath, outStackPath):
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath)
### setup Xmipp command
xmippexe = apParam.getExecPath("xmipp_normalize", die=True)
apDisplay.printMsg("Using Xmipp to normalize particle stack")
normtime = time.time()
xmippopts = (" " +
" -i %s" % os.path.join(self.params['rundir'], selfile) +
" -method Ramp " + " -background circle %i" %
(self.stack['boxsize'] / self.params['bin'] * 0.4) +
" -remove_black_dust" + " -remove_white_dust" +
" -thr_black_dust -%.2f" % (self.params['xmipp-norm']) +
" -thr_white_dust %.2f" % (self.params['xmipp-norm']))
xmippcmd = xmippexe + " " + xmippopts
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
normtime = time.time() - normtime
apDisplay.printMsg("Xmipp normalization time: " +
apDisplay.timeString(normtime))
### recombine particles to a single imagic stack
tmpstack = "tmp.xmippStack.hed"
apXmipp.gatherSingleFilesIntoStack(selfile, tmpstack)
apFile.moveStack(tmpstack, outStackPath)
### clean up directory
apFile.removeFile(selfile)
apFile.removeDir("partfiles")
def createOrientationReference(self):
"""
convert spider orientation reference
"""
orientref = os.path.join(self.params['rundir'], "orient.spi")
apFile.removeFile(orientref, warn=True)
#templatedata = apTemplate.getTemplateFromId(self.params['orientref'])
#templatefile = os.path.join(templatedata['path']['path'], templatedata['templatename'])
### hack to use standard filtering library
templateparams = {}
templateparams['apix'] = self.stack['apix']
templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
templateparams['templateIds'] = [self.params['orientref'],]
templateparams['bin'] = self.params['bin']
templateparams['lowpass'] = self.params['lowpass']
templateparams['median'] = None
templateparams['pixlimit'] = None
print 'Converting orientation reference:\n', templateparams
apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
filelist = apTemplate.getTemplates(templateparams)
mrcfile = filelist[0]
localclip = self.clipsize/self.params['bin']
emancmd = ("proc2d templates/"+mrcfile+" "+orientref
+" clip="+str(localclip)+","+str(localclip)
+" edgenorm spiderswap-single ")
if self.params['inverttemplates'] is True:
emancmd += " invert "
apEMAN.executeEmanCmd(emancmd, showcmd=False)
return orientref
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.checkNoRefRun()
### convert stack to spider
spiderstack = self.createSpiderFile()
### create initialization template
if self.params['initmethod'] == 'allaverage':
templatefile = self.averageTemplate()
elif self.params['initmethod'] == 'selectrand':
templatefile = self.selectRandomParticles()
elif self.params['initmethod'] == 'randpart':
templatefile = self.pickRandomParticle()
elif self.params['initmethod'] == 'template':
templatefile = self.getTemplate()
else:
apDisplay.printError("unknown initialization method defined: "
+str(self.params['initmethod'])+" not in "+str(self.initmethods))
apDisplay.printColor("Running spider this can take awhile","cyan")
### run the alignment
aligntime = time.time()
pixrad = int(round(self.params['partrad']/self.stack['apix']/self.params['bin']))
alignedstack, self.partlist = alignment.refFreeAlignParticles(
spiderstack, templatefile,
self.params['numpart'], pixrad,
self.params['firstring'], self.params['lastring'],
rundir = ".")
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=False)
### convert stack to imagic
imagicstack = self.convertSpiderStack(alignedstack)
apFile.removeFile(alignedstack)
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = aligntime
self.insertNoRefRun(imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def checkAndCopyFiles(self):
if not os.path.isfile(self.params['inputstack']):
apDisplay.printError("Stack file %s does not exist"%(self.params['inputstack']))
if not os.path.isfile(self.params['inputvol']):
apDisplay.printError("Initial model file %s does not exist"%(self.params['inputvol']))
if not os.path.isfile(self.params['inputparam']):
apDisplay.printError("Initial parameter file %s does not exist"%(self.params['inputparam']))
apParam.createDirectory(self.params['rundir'])
firstVolFile = os.path.join(self.params['rundir'], "initial_volume.mrc")
apDisplay.printColor("Linking initial model %s to recon folder %s"%
(self.params['inputvol'], firstVolFile), "purple")
if not os.path.exists(firstVolFile):
os.symlink(self.params['inputvol'], firstVolFile)
#shutil.copy(self.params['inputvol'], firstVolFile)
firstParamFile = os.path.join(self.params['rundir'], "initial_params.par")
apDisplay.printColor("Copying initial parameters %s to recon folder %s"
%(self.params['inputparam'], firstParamFile), "purple")
apFile.removeFile(firstParamFile)
time.sleep(0.1)
shutil.copy(self.params['inputparam'], firstParamFile)
os.chdir(self.params['rundir'])
for side in ('left', 'right'):
prefix = "%s.iter%02d"%(side, 0)
volFile = "threed.%s.mrc"%(prefix)
if not os.path.islink(volFile):
os.symlink(os.path.basename(firstVolFile), volFile)
self.createSplitParamFiles(firstParamFile)
return
def createTemplateStack(self):
"""
convert spider template stack
"""
templatestack = os.path.join(self.params['rundir'], "templatestack.spi")
apFile.removeFile(templatestack, warn=True)
### hack to use standard filtering library
templateparams = {}
templateparams['apix'] = self.stack['apix']
templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
templateparams['templateIds'] = self.templatelist
templateparams['bin'] = self.params['bin']
templateparams['lowpass'] = self.params['lowpass']
templateparams['median'] = None
templateparams['pixlimit'] = None
print 'Converting reference templates:\n', templateparams
apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
filelist = apTemplate.getTemplates(templateparams)
localclip = self.clipsize/self.params['bin']
for mrcfile in filelist:
emancmd = ("proc2d templates/"+mrcfile+" "+templatestack
+" clip="+str(localclip)+","+str(localclip)
+" edgenorm spiderswap ")
if self.params['inverttemplates'] is True:
emancmd += " invert "
apEMAN.executeEmanCmd(emancmd, showcmd=False)
return templatestack
def stackToSpiderStack(stack,spiderstack,apix,boxsize,lp=0,hp=0,bin=1,numpart=0):
"""
convert an input stack (i.e. imagic) and write it out to a spider-formatted stack
"""
emancmd = "proc2d "
if not os.path.isfile(stack):
apDisplay.printError("stackfile does not exist: "+stack)
emancmd += stack+" "
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "apix="+str(apix)+" "
if lp > 0:
emancmd += "lp="+str(lp)+" "
if hp > 0:
emancmd += "hp="+str(hp)+" "
if bin > 1:
clipboxsize = boxsize*bin
emancmd += "shrink="+str(bin)+" "
emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" "
if numpart > 0:
emancmd += "last="+str(numpart-1)+" "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
from appionlib import apEMAN
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def convertPostscriptToPng(psfile, pngfile, size=1024):
### better pstopnm pre-step
pstopnmcmd = "pstopnm -xsize=2000 -ysize=2000 -xborder=0 -yborder=0 -portrait "+psfile
proc = subprocess.Popen(pstopnmcmd, shell=True)
proc.wait()
### direct conversion
ppmfile = os.path.splitext(psfile)[0]+"001.ppm"
if os.path.isfile(ppmfile):
imagemagickcmd = ("convert -colorspace Gray -trim -resize "
+str(size)+"x"+str(size)+" "+ppmfile+" "+pngfile)
else:
ppmfile = psfile+"001.ppm"
if os.path.isfile(ppmfile):
imagemagickcmd = ("convert -colorspace Gray -trim -resize "
+str(size)+"x"+str(size)+" "+ppmfile+" "+pngfile)
else:
imagemagickcmd = ("convert -colorspace Gray -trim -resize "
+str(size)+"x"+str(size)+" "+psfile+" "+pngfile)
proc = subprocess.Popen(imagemagickcmd, shell=True)
proc.wait()
if os.path.isfile(ppmfile):
apFile.removeFile(ppmfile)
if not os.path.isfile(pngfile):
apDisplay.printWarning("Postscript image conversion failed")
def processTemplate(self, mrcfile):
### shrink
apDisplay.printMsg("Binning template by a factor of " +
str(self.params['bin']))
clipsize = int(
math.floor(self.stack['boxsize'] / self.params['bin']) *
self.params['bin'])
emancmd = ("proc2d " + mrcfile + " " + mrcfile + " shrink=" +
str(self.params['bin']) + " spiderswap ")
emancmd += "clip=" + str(clipsize) + "," + str(clipsize) + " "
apEMAN.executeEmanCmd(emancmd)
### normalize and center
#apDisplay.printMsg("Normalize and centering template")
#emancmd = "proc2d "+mrcfile+" "+mrcfile+" edgenorm center"
#apEMAN.executeEmanCmd(emancmd)
### convert to SPIDER
apDisplay.printMsg("Converting template to SPIDER")
templatefile = "template.spi"
if os.path.isfile(templatefile):
apFile.removeFile(templatefile, warn=True)
emancmd = "proc2d template.mrc " + templatefile + " spiderswap "
apEMAN.executeEmanCmd(emancmd)
return templatefile
def convertSpiderStack(self, spiderstack):
"""
takes the stack file and creates a spider file ready for processing
"""
emancmd = "proc2d "
if not os.path.isfile(spiderstack):
apDisplay.printError("stackfile does not exist: " + spiderstack)
emancmd += spiderstack + " "
imagicstack = os.path.join(self.params['rundir'], "alignstack.hed")
apFile.removeFile(imagicstack, warn=True)
emancmd += imagicstack + " "
starttime = time.time()
apDisplay.printColor(
"Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor(
"finished eman in " +
apDisplay.timeString(time.time() - starttime), "cyan")
numpart = apFile.numImagesInStack(imagicstack)
if numpart != self.params['numpart']:
apDisplay.printError(
"The number of particles returned is different from the number requested"
)
return imagicstack
def makeEvenOddClasses(self, listfile, outputstack, classdata, maskrad):
f=open(listfile,'r')
f.readline()
lines = f.readlines()
f.close()
randstr = str(int(random.random()*10e5))
evenfile = self.rootname+"-even.lst"
evenf = open(evenfile,'w')
oddfile = self.rootname+"-odd.lst"
oddf = open(oddfile,'w')
evenf.write("#LST\n")
oddf.write("#LST\n")
neven=0
nodd=0
for i in range(0, len(lines)):
if i%2:
nodd+=1
oddf.write(lines[i])
else:
neven+=1
evenf.write(lines[i])
evenf.close()
oddf.close()
if neven>0:
self.makeClassAverages(evenfile, self.params['evenstack'], classdata, maskrad)
if nodd>0:
self.makeClassAverages(oddfile, self.params['oddstack'], classdata, maskrad)
apFile.removeFile(evenfile)
apFile.removeFile(oddfile)
def prepareStack(self):
### setup box and mask sizes
self.alignstackdata = self.getAlignedStack()
maskpixrad = self.params['maskrad'] / self.alignstackdata[
'pixelsize'] / self.params['bin']
boxpixdiam = int(math.ceil(maskpixrad) + 1) * 2
if boxpixdiam * self.params['bin'] > self.alignstackdata['boxsize']:
boxpixdiam = math.floor(self.alignstackdata['boxsize'] /
self.params['bin'])
clippixdiam = boxpixdiam * self.params['bin']
apDisplay.printMsg("Pixel mask radius=" + str(maskpixrad))
### convert aligned stack to local dir
oldalignedstack = os.path.join(self.alignstackdata['path']['path'],
self.alignstackdata['imagicfile'])
alignedstackname = "alignedstack.hed"
alignedstack = os.path.join(self.params['rundir'], alignedstackname)
apFile.removeFile(alignedstack)
emancmd = ("proc2d %s %s shrink=%d clip=%d,%d" %
(oldalignedstack, alignedstack, self.params['bin'],
clippixdiam, clippixdiam))
if self.params['numpart'] is not None and self.params[
'numpart'] < self.numpart:
emancmd += " last=%d" % (self.params['numpart'] - 1)
self.numpart = self.params['numpart']
apEMAN.executeEmanCmd(emancmd, verbose=True)
return alignedstack
def backProjection(stack,
select,
ang,
out,
sym=None,
nproc=1,
dataext=".spi",
inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(nproc=nproc,
dataext=dataext,
logo=False,
log=False)
else:
mySpi = inMySpi
apFile.removeFile(out)
if sym is None:
sym = "*"
mySpi.toSpiderQuiet("BP 3F",
spyder.fileFilter(stack) + "@******",
spyder.fileFilter(select), spyder.fileFilter(ang),
spyder.fileFilter(sym), spyder.fileFilter(out))
if inMySpi is False:
mySpi.close()
def uploadImages(self):
### get simulated instrument ids
scopeid, cameraid = self.getInstrumentIds()
### Download images
imglist = self.downloadImagesFromAMI()
### create batch file
imagedatfile = self.createImageBatchFile(imglist)
### run command
script = os.path.join(self.appiondir, "bin", "imageloader.py ")
params = (
" --runname=%s --projectid=%d --session=%s --batch=%s --scopeid=%d --cameraid=%d --description='%s' "
% (
self.sessionname,
self.params["projectid"],
self.sessionname,
imagedatfile,
scopeid,
cameraid,
self.params["description"],
)
)
if self.params["commit"] is True:
params += " --commit "
else:
params += " --no-commit "
self.runCommand(script + " " + params)
for imgname in imglist:
apFile.removeFile(imgname)
def start(self):
#get aligned stack id
aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
xSizeVoxel = aligndata['boxsize']
#get averaged image
avgmrc = os.path.join(aligndata['path']['path'], aligndata["avgmrcfile"])
avg = mrc.read(avgmrc)
tmpSpiderFile="average.xmp"
spider.write(avg, tmpSpiderFile)
self.runFindCenter(tmpSpiderFile,xSizeVoxel)
#get aligned stack
alignStack = os.path.join(aligndata['path']['path'], aligndata["imagicfile"])
tempFileNameforSpectra=self.runMakeSpectra(alignStack)
#kerdensom will work with spectra output
self.runKerdenSOM(tempFileNameforSpectra)
self.createMontageInMemory()
self.insertRotKerDenSOM()
#apFile.removeFile(outdata)
apFile.removeFilePattern("*.cod")
apFile.removeFilePattern("*.err")
apFile.removeFilePattern("*.his")
apFile.removeFilePattern("*.inf")
apFile.removeFilePattern("*.vs")
apFile.removeFilePattern("*.xmp")
apFile.removeFile(tempFileNameforSpectra)
def peakTreeToPikFile(peaktree, imgname, tmpl, rundir="."):
outpath = os.path.join(rundir, "pikfiles")
apParam.createDirectory(outpath, warning=False)
outfile = os.path.join(outpath, imgname + "." + str(tmpl) + ".pik")
apFile.removeFile(outfile, warn=True)
#WRITE PIK FILE
f = open(outfile, 'w')
f.write(
"#filename x y mean stdev corr_coeff peak_size templ_num angle moment diam\n"
)
for peakdict in peaktree:
row = peakdict['ycoord']
col = peakdict['xcoord']
if 'corrcoeff' in peakdict:
rho = peakdict['corrcoeff']
else:
rho = 1.0
size = peakdict['peakarea']
mean_str = "%.4f" % peakdict['correlation']
std_str = "%.4f" % peakdict['peakstddev']
mom_str = "%.4f" % peakdict['peakmoment']
if peakdict['diameter'] is not None:
diam_str = "%.2f" % peakdict['diameter']
else:
diam_str = "0"
if 'template' in peakdict:
tmplnum = peakdict['template']
else:
tmplnum = tmpl
#filename x y mean stdev corr_coeff peak_size templ_num angle moment
out = imgname+".mrc "+str(int(col))+" "+str(int(row))+ \
" "+mean_str+" "+std_str+" "+str(rho)+" "+str(int(size))+ \
" "+str(tmplnum)+" 0 "+mom_str+" "+diam_str
f.write(str(out) + "\n")
f.close()
def ClCla(alignedstack, numpart=None, numclasses=40,
factorlist=range(1,5), corandata="coran/corandata", dataext=".spi"):
"""
this doesn't work
"""
if alignedstack[-4:] == dataext:
alignedstack = alignedstack[:-4]
rundir = "cluster"
classavg = rundir+"/"+("classavgstack%03d" % numclasses)
classvar = rundir+"/"+("classvarstack%03d" % numclasses)
apParam.createDirectory(rundir)
for i in range(numclasses):
apFile.removeFile(rundir+("/classdoc%04d" % (i+1))+dataext)
apFile.removeFile(rundir+"/clusterdoc"+dataext)
factorstr, factorkey = operations.intListToString(factorlist)
### do hierarchical clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider(
"CL CLA",
corandata, # path to coran data
rundir+"/clusterdoc", #clusterdoc file
factorstr, #factor numbers
"5,8",
"4",
"2", # minimum number of particles per class
"Y", rundir+"/dendrogram.ps",
"Y", rundir+"/dendrogramdoc",
)
mySpider.close()
def symmetryDoc(symtype, symfold=None, outfile="sym.spi", dataext=".spi"):
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
apFile.removeFile(outfile)
mySpi.toSpiderQuiet("SY", spyder.fileFilter(outfile), symtype)
if symfold is not None:
mySpi.toSpiderQuiet(symfold)
mySpi.close()
def backprojectCG(stackfile, eulerdocfile, volfile, numpart, pixrad, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
### setup
starttime = time.time()
stackfile = spyder.fileFilter(stackfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(stackfile+dataext):
apDisplay.printError("stack file not found: "+stackfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(volfile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("BP CG",
stackfile+"@*****", #stack file
"1-%d"%(numpart), #number of particles
str(pixrad), #particle radius
eulerdocfile, #angle doc file
"N", #has symmetry?, does not work
volfile, #filename for volume
"%.1e,%.1f" % (1.0e-5, 0.0), #error, chi^2 limits
"%d,%d" % (25,1), #iterations, 1st derivative mode
"2000", #lambda - higher=less sensitive to noise
)
mySpider.close()
apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def processParticles(self, imgdata, partdatas, shiftdata):
# need to avoid non-frame saved image for proper caching
if imgdata is None or imgdata['camera']['save frames'] != True:
apDisplay.printWarning('%s skipped for no-frame-saved\n ' %
imgdata['filename'])
return
### first remove any existing files
rundir = self.params['rundir']
imgname = imgdata['filename']
moviegrouppath = os.path.join(rundir,
'%s_*.%s' % (imgname, self.movieformat))
apFile.removeFile(moviegrouppath)
shortname = apDisplay.short(imgdata['filename'])
apFile.removeFile(os.path.join(rundir, '%s*.jpg' % (shortname)))
### set processing image
try:
self.dd.setImageData(imgdata)
except Exception, e:
raise
apDisplay.printWarning('%s: %s' % (e.__class__.__name__, e))
return
def assemblePngs(keys, tag, montagestack):
apDisplay.printMsg("assembling pngs into montage")
# get path from montagestack
stackpath = os.path.dirname(os.path.abspath(montagestack))
montagecmd = "montage -geometry +4+4 "
montagestackdata = apImagicFile.readImagic(montagestack)
for i,key in enumerate(keys):
if i % 20 == 0:
sys.stderr.write(".")
pngfile = key+".png"
array = montagestackdata['images'][i]
apImage.arrayToPng(array, pngfile, normalize=True, msg=False)
#proccmd = "proc2d "+montagestack+" "+pngfile+" first="+str(i)+" last="+str(i)
#apEMAN.executeEmanCmd(proccmd, verbose=False, showcmd=False)
montagecmd += pngfile+" "
apDisplay.printMsg("montaging")
montagefile = os.path.join(stackpath,"montage"+tag+".png")
montagecmd += montagefile
apEMAN.executeEmanCmd(montagecmd, verbose=True)
#rotatecmd = "mogrify -rotate 180 -flop "+montagefile
#apEMAN.executeEmanCmd(rotatecmd, verbose=False)
for key in keys:
apFile.removeFile(key+".png")
if not os.path.isfile(montagefile):
apDisplay.printWarning("failed to create montage file")
def projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext=".spi"):
"""
project 3D volumes using given Euler angles
"""
starttime = time.time()
volfile = spyder.fileFilter(volfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
projstackfile = spyder.fileFilter(projstackfile)
if not os.path.isfile(volfile+dataext):
apDisplay.printError("volume file not found: "+volfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(projstackfile)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("PJ 3Q",
volfile, #input vol file
str(pixrad), #pixel radius
"1-%d"%(numpart), #number of particles
eulerdocfile, #Euler DOC file
projstackfile+"@*****", #output projections
)
mySpider.close()
apDisplay.printColor("finished projections in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def peakTreeToPikFile(peaktree, imgname, tmpl, rundir="."):
outpath = os.path.join(rundir, "pikfiles")
apParam.createDirectory(outpath, warning=False)
outfile = os.path.join(outpath, imgname+"."+str(tmpl)+".pik")
apFile.removeFile(outfile, warn=True)
#WRITE PIK FILE
f=open(outfile, 'w')
f.write("#filename x y mean stdev corr_coeff peak_size templ_num angle moment diam\n")
for peakdict in peaktree:
row = peakdict['ycoord']
col = peakdict['xcoord']
if 'corrcoeff' in peakdict:
rho = peakdict['corrcoeff']
else:
rho = 1.0
size = peakdict['peakarea']
mean_str = "%.4f" % peakdict['correlation']
std_str = "%.4f" % peakdict['peakstddev']
mom_str = "%.4f" % peakdict['peakmoment']
if peakdict['diameter'] is not None:
diam_str = "%.2f" % peakdict['diameter']
else:
diam_str = "0"
if 'template' in peakdict:
tmplnum = peakdict['template']
else:
tmplnum = tmpl
#filename x y mean stdev corr_coeff peak_size templ_num angle moment
out = imgname+".mrc "+str(int(col))+" "+str(int(row))+ \
" "+mean_str+" "+std_str+" "+str(rho)+" "+str(int(size))+ \
" "+str(tmplnum)+" 0 "+mom_str+" "+diam_str
f.write(str(out)+"\n")
f.close()
def processImage(self, imgdata): self.ctfvalues = None fftpath = os.path.join(self.powerspecdir, apDisplay.short(imgdata['filename'])+'.powerspec.mrc') self.processAndSaveFFT(imgdata, fftpath) self.runRefineCTF(imgdata, fftpath) apFile.removeFile(fftpath) return
def createSpiderFile(self):
"""
takes the stack file and creates a spider file ready for processing
"""
emancmd = "proc2d "
if not os.path.isfile(self.stack['file']):
apDisplay.printError("stackfile does not exist: "+self.stack['file'])
emancmd += self.stack['file']+" "
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "apix="+str(self.stack['apix'])+" "
if self.params['lowpass'] > 0:
emancmd += "lp="+str(self.params['lowpass'])+" "
if self.params['highpass'] > 0:
emancmd += "hp="+str(self.params['highpass'])+" "
if self.params['bin'] > 1:
clipboxsize = self.boxsize*self.params['bin']
emancmd += "shrink="+str(self.params['bin'])+" "
emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" "
emancmd += "last="+str(self.params['numpart']-1)+" "
emancmd += "spider edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
return spiderstack
def convertPDBtoMRC(self, pdbfile):
### create density from pdb
spidername = self.params['name']+".spi"
mrcname = self.params['name']+".mrc"
if self.params['method'] == 'spider':
if self.params['bunit'] is True:
pdbfile = self.removeModelFlags(pdbfile)
volFun.pdb2vol(pdbfile, self.params['apix'], self.params['boxsize'], spidername)
if not os.path.isfile(spidername):
apDisplay.printError("SPIDER did not create density file: "+spidername)
### convert spider to mrc format
apDisplay.printMsg("converting spider file to mrc")
emancmd='proc3d %s %s apix=%.4f lp=%.2f norm=0,1 origin=0,0,0' % (spidername, mrcname,
self.params['apix'], self.params['lowpass'])
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
apFile.removeFile(spidername)
elif self.params['method'] == 'eman':
mult = 1.0
### use resolution of mult*apix, because this is slow and lowpass is fast
emancmd='pdb2mrc %s %s apix=%.4f res=%.2f box=%d center allmdl' % (pdbfile, mrcname,
self.params['apix'], mult*self.params['apix'], self.params['boxsize'])
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
### lowpass to desired resolution minus the res factor specified
lp = math.sqrt(self.params['lowpass']**2 - (mult*self.params['apix'])**2)
emancmd='proc3d %s %s apix=%.4f lp=%.2f norm=0,1 origin=0,0,0' % (mrcname, mrcname,
self.params['apix'], lp)
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
if self.params['viper2eman'] is True:
apVolume.viper2eman(mrcname, mrcname, apix=self.params['apix'])
return mrcname
def symmetryDoc(symtype,symfold=None,outfile="sym.spi",dataext=".spi"):
mySpi=spyder.SpiderSession(dataext=dataext,logo=False,log=False)
apFile.removeFile(outfile)
mySpi.toSpiderQuiet("SY",spyder.fileFilter(outfile),symtype)
if symfold is not None:
mySpi.toSpiderQuiet(symfold)
mySpi.close()
def convertPDBtoMRC(self, pdbfile):
### create density from pdb
spidername = self.params['name']+".spi"
mrcname = self.params['name']+".mrc"
if self.params['method'] == 'spider':
if self.params['bunit'] is True:
pdbfile = self.removeModelFlags(pdbfile)
volFun.pdb2vol(pdbfile, self.params['apix'], self.params['boxsize'], spidername)
if not os.path.isfile(spidername):
apDisplay.printError("SPIDER did not create density file: "+spidername)
### convert spider to mrc format
apDisplay.printMsg("converting spider file to mrc")
emancmd='proc3d %s %s apix=%.4f lp=%.2f norm=0,1 origin=0,0,0' % (spidername, mrcname,
self.params['apix'], self.params['lowpass'])
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
apFile.removeFile(spidername)
elif self.params['method'] == 'eman':
mult = 1.0
### use resolution of mult*apix, because this is slow and lowpass is fast
emancmd='pdb2mrc %s %s apix=%.4f res=%.2f box=%d center allmdl' % (pdbfile, mrcname,
self.params['apix'], mult*self.params['apix'], self.params['boxsize'])
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
### lowpass to desired resolution minus the res factor specified
lp = math.sqrt(self.params['lowpass']**2 - (mult*self.params['apix'])**2)
emancmd='proc3d %s %s apix=%.4f lp=%.2f norm=0,1 origin=0,0,0' % (mrcname, mrcname,
self.params['apix'], lp)
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
if self.params['viper2eman'] is True:
apVolume.viper2eman(mrcname, mrcname, apix=self.params['apix'])
return mrcname
def xmippNormStack(self, inStackPath, outStackPath):
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath)
### setup Xmipp command
xmippexe = apParam.getExecPath("xmipp_normalize", die=True)
apDisplay.printMsg("Using Xmipp to normalize particle stack")
normtime = time.time()
xmippopts = ( " "
+" -i %s"%os.path.join(self.params['rundir'],selfile)
+" -method Ramp "
+" -background circle %i"%(self.stack['boxsize']/self.params['bin']*0.4)
+" -remove_black_dust"
+" -remove_white_dust"
+" -thr_black_dust -%.2f"%(self.params['xmipp-norm'])
+" -thr_white_dust %.2f"%(self.params['xmipp-norm'])
)
xmippcmd = xmippexe+" "+xmippopts
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
normtime = time.time() - normtime
apDisplay.printMsg("Xmipp normalization time: "+apDisplay.timeString(normtime))
### recombine particles to a single imagic stack
tmpstack = "tmp.xmippStack.hed"
apXmipp.gatherSingleFilesIntoStack(selfile,tmpstack)
apFile.moveStack(tmpstack,outStackPath)
### clean up directory
apFile.removeFile(selfile)
apFile.removeDir("partfiles")
def backproject3F(stackfile, eulerdocfile, volfile, numpart, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
### setup
starttime = time.time()
stackfile = spyder.fileFilter(stackfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(stackfile+dataext):
apDisplay.printError("stack file not found: "+stackfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(volfile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("BP 3F",
stackfile+"@*****", #stack file
"1-%d"%(numpart), #number of particles
eulerdocfile, #angle doc file
"*", #input symmetry file, '*' for skip
volfile, #filename for volume
)
mySpider.close()
apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def start(self):
#get aligned stack id
aligndata = appiondata.ApAlignStackData.direct_query(
self.params['alignstackid'])
xSizeVoxel = aligndata['boxsize']
#get averaged image
avgmrc = os.path.join(aligndata['path']['path'],
aligndata["avgmrcfile"])
avg = mrc.read(avgmrc)
tmpSpiderFile = "average.xmp"
spider.write(avg, tmpSpiderFile)
self.runFindCenter(tmpSpiderFile, xSizeVoxel)
#get aligned stack
alignStack = os.path.join(aligndata['path']['path'],
aligndata["imagicfile"])
tempFileNameforSpectra = self.runMakeSpectra(alignStack)
#kerdensom will work with spectra output
self.runKerdenSOM(tempFileNameforSpectra)
self.createMontageInMemory()
self.insertRotKerDenSOM()
#apFile.removeFile(outdata)
apFile.removeFilePattern("*.cod")
apFile.removeFilePattern("*.err")
apFile.removeFilePattern("*.his")
apFile.removeFilePattern("*.inf")
apFile.removeFilePattern("*.vs")
apFile.removeFilePattern("*.xmp")
apFile.removeFile(tempFileNameforSpectra)
def backProjection( stack, select, ang, out, sym=None, nproc=1, dataext=".spi", inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(nproc=nproc,dataext=dataext,logo=False,log=False) else: mySpi=inMySpi apFile.removeFile(out) if sym is None: sym="*" mySpi.toSpiderQuiet( "BP 3F", spyder.fileFilter(stack)+"@******", spyder.fileFilter(select), spyder.fileFilter(ang), spyder.fileFilter(sym), spyder.fileFilter(out) ) if inMySpi is False: mySpi.close()
def createSpiderFile(self):
"""
takes the stack file and creates a spider file ready for processing
"""
emancmd = "proc2d "
if not os.path.isfile(self.stack["file"]):
apDisplay.printError("stackfile does not exist: " + self.stack["file"])
emancmd += self.stack["file"] + " "
spiderstack = os.path.join(self.params["rundir"], "start.spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack + " "
emancmd += "apix=" + str(self.stack["apix"]) + " "
if self.params["lowpass"] > 0:
emancmd += "lp=" + str(self.params["lowpass"]) + " "
emancmd += "last=" + str(self.params["numpart"] - 1) + " "
emancmd += "shrink=" + str(self.params["bin"]) + " "
clipsize = int(math.floor(self.stack["boxsize"] / self.params["bin"] / 2.0) * self.params["bin"] * 2)
emancmd += "clip=" + str(clipsize) + "," + str(clipsize) + " "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in " + apDisplay.timeString(time.time() - starttime), "cyan")
return spiderstack
def start(self):
aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
boxsize = aligndata['boxsize']
apix = aligndata['pixelsize']
maskpixrad = self.params['maskrad']/apix
if maskpixrad*2 > boxsize-2:
apDisplay.printError("Mask radius is too big for boxsize: %d > %d"%(maskpixrad*2,boxsize-2))
apDisplay.printMsg("Mask radius and boxsize: %.1f < %d"%(maskpixrad*2,boxsize-2))
self.instack = os.path.join(aligndata['path']['path'], aligndata['imagicfile'])
outdata = "stack.data"
apXmipp.convertStackToXmippData(self.instack, outdata, maskpixrad,
boxsize, numpart=self.params['numpart']-1)
self.runKerdenSOM(outdata)
if apFile.stackSize(self.instack) > 3.0*(1024**3):
# Big stacks use eman
self.createMontageByEMAN()
binned = None
else:
binned = self.createMontageInMemory(apix)
self.insertKerDenSOM(binned=binned)
apFile.removeFile(outdata)
apFile.removeFilePattern("*.cod")
def rctParticleShift(volfile, origstackfile, eulerdocfile, iternum, numpart, pixrad, timestamp, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
starttime = time.time()
### create corresponding projections
projstackfile = "projstack%s-%03d.spi"%(timestamp, iternum)
projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext)
### clean up files
ccdocfile = "ccdocfile%s-%03d.spi"%(timestamp, iternum)
apFile.removeFile(ccdocfile)
alignstackfile = "alignstack%s-%03d.spi"%(timestamp, iternum)
apFile.removeFile(alignstackfile)
### align particles to projection
apDisplay.printMsg("Shifting particles")
crossCorrelateAndShift(origstackfile, projstackfile, alignstackfile, ccdocfile, numpart)
if not os.path.isfile(alignstackfile):
apDisplay.printError("aligned stack file not found: "+alignstackfile)
apDisplay.printColor("finished correlations in "+apDisplay.timeString(time.time()-starttime), "cyan")
return alignstackfile
def convertStackToSpider(self, emanstackfile, classnum):
"""
takes the stack file and creates a spider file ready for processing
"""
if not os.path.isfile(emanstackfile):
apDisplay.printError("stackfile does not exist: "+emanstackfile)
### first high pass filter particles
apDisplay.printMsg("pre-filtering particles")
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])
emancmd = ("proc2d "+emanstackfile+" "+emanstackfile
+" apix="+str(apix)+" hp="+str(self.params['highpasspart'])
+" inplace")
apEMAN.executeEmanCmd(emancmd, verbose=True)
### convert imagic stack to spider
emancmd = "proc2d "
emancmd += emanstackfile+" "
spiderstack = os.path.join(self.params['rundir'], str(classnum), "otrstack"+self.timestamp+".spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "spiderswap edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
return spiderstack
def runRmeasure(self, volfile):
emancmd = "proc3d %s %s"%(volfile, "rmeasure.mrc")
apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True)
apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin']
self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc")
#apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan")
apFile.removeFile("rmeasure.mrc")
def createTemplateStack(self):
"""
takes the spider file and creates an average template of all particles
"""
templatestack = os.path.join(self.params['rundir'], "templatestack00.spi")
apFile.removeFile(templatestack, warn=True)
### hack to use standard filtering library
templateparams = {}
templateparams['apix'] = self.stack['apix']
templateparams['rundir'] = os.path.join(self.params['rundir'], "templates")
templateparams['templateIds'] = self.templatelist
templateparams['bin'] = self.params['bin']
templateparams['lowpass'] = self.params['lowpass']
templateparams['median'] = None
templateparams['pixlimit'] = None
print templateparams
apParam.createDirectory(os.path.join(self.params['rundir'], "templates"))
filelist = apTemplate.getTemplates(templateparams)
for mrcfile in filelist:
emancmd = ("proc2d templates/"+mrcfile+" "+templatestack
+" clip="+str(self.boxsize)+","+str(self.boxsize)
+" spider ")
if self.params['inverttemplates'] is True:
emancmd += " invert "
apEMAN.executeEmanCmd(emancmd, showcmd=False)
return templatestack
def makeEvenOddClasses(self, listfile, outputstack, classdata, maskrad):
f = open(listfile, 'r')
f.readline()
lines = f.readlines()
f.close()
randstr = str(int(random.random() * 10e5))
evenfile = self.rootname + "-even.lst"
evenf = open(evenfile, 'w')
oddfile = self.rootname + "-odd.lst"
oddf = open(oddfile, 'w')
evenf.write("#LST\n")
oddf.write("#LST\n")
neven = 0
nodd = 0
for i in range(0, len(lines)):
if i % 2:
nodd += 1
oddf.write(lines[i])
else:
neven += 1
evenf.write(lines[i])
evenf.close()
oddf.close()
if neven > 0:
self.makeClassAverages(evenfile, self.params['evenstack'],
classdata, maskrad)
if nodd > 0:
self.makeClassAverages(oddfile, self.params['oddstack'], classdata,
maskrad)
apFile.removeFile(evenfile)
apFile.removeFile(oddfile)
def createSpiderFile(self):
"""
takes the stack file and creates a spider file ready for processing
"""
emancmd = "proc2d "
if not os.path.isfile(self.stack['file']):
apDisplay.printError("stackfile does not exist: "+self.stack['file'])
emancmd += self.stack['file']+" "
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apFile.removeFile(spiderstack, warn=True)
emancmd += spiderstack+" "
emancmd += "apix="+str(self.stack['apix'])+" "
if self.params['lowpass'] > 0:
emancmd += "lp="+str(self.params['lowpass'])+" "
if self.params['highpass'] > 0:
emancmd += "hp="+str(self.params['highpass'])+" "
if self.params['bin'] > 1:
clipboxsize = self.boxsize*self.params['bin']
emancmd += "shrink="+str(self.params['bin'])+" "
emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" "
emancmd += "last="+str(self.params['numpart']-1)+" "
emancmd += "spider edgenorm"
starttime = time.time()
apDisplay.printColor("Running spider stack conversion this can take a while", "cyan")
apEMAN.executeEmanCmd(emancmd, verbose=True)
apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan")
return spiderstack
def createTemplateStack(self):
"""
convert spider template stack
"""
templatestack = os.path.join(self.params['rundir'],
"templatestack.spi")
apFile.removeFile(templatestack, warn=True)
### hack to use standard filtering library
templateparams = {}
templateparams['apix'] = self.stack['apix']
templateparams['rundir'] = os.path.join(self.params['rundir'],
"templates")
templateparams['templateIds'] = self.templatelist
templateparams['bin'] = self.params['bin']
templateparams['lowpass'] = self.params['lowpass']
templateparams['median'] = None
templateparams['pixlimit'] = None
print 'Converting reference templates:\n', templateparams
apParam.createDirectory(
os.path.join(self.params['rundir'], "templates"))
filelist = apTemplate.getTemplates(templateparams)
localclip = self.clipsize / self.params['bin']
for mrcfile in filelist:
emancmd = ("proc2d templates/" + mrcfile + " " + templatestack +
" clip=" + str(localclip) + "," + str(localclip) +
" edgenorm spiderswap ")
if self.params['inverttemplates'] is True:
emancmd += " invert "
apEMAN.executeEmanCmd(emancmd, showcmd=False)
return templatestack
def ClCla(alignedstack, numpart=None, numclasses=40,
factorlist=range(1,5), corandata="coran/corandata", dataext=".spi"):
"""
this doesn't work
"""
if alignedstack[-4:] == dataext:
alignedstack = alignedstack[:-4]
rundir = "cluster"
classavg = rundir+"/"+("classavgstack%03d" % numclasses)
classvar = rundir+"/"+("classvarstack%03d" % numclasses)
apParam.createDirectory(rundir)
for i in range(numclasses):
apFile.removeFile(rundir+("/classdoc%04d" % (i+1))+dataext)
apFile.removeFile(rundir+"/clusterdoc"+dataext)
factorstr, factorkey = operations.intListToString(factorlist)
### do hierarchical clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider(
"CL CLA",
corandata, # path to coran data
rundir+"/clusterdoc", #clusterdoc file
factorstr, #factor numbers
"5,8",
"4",
"2", # minimum number of particles per class
"Y", rundir+"/dendrogram.ps",
"Y", rundir+"/dendrogramdoc",
)
mySpider.close()
def undoCTFCorrect(self, newstackroot):
# At this point, the stack needs to be remade un-ctf-corrected, and possibly normalized and/or inverted
apDisplay.printWarning('Relion needs a stack without ctf correction. A new stack is being made....')
# Gather all the makestack parameters
totalpart = self.originalStackData.numpart
numpart = totalpart if not self.params['last'] else min(self.params['last'],totalpart)
stackpathname = os.path.basename( self.originalStackData.path )
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir,'start.hed')
presetname = self.originalStackData.preset
# binning is combination of the original binning of the stack and the preparation binnning
bin = self.originalStackData.bin * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin
lowpasstext = setArgText( 'lowpass', ( self.params['lowpass'], self.originalStackData.lowpass ), False)
highpasstext = setArgText( 'highpass', ( self.params['highpass'], self.originalStackData.highpass ), True)
partlimittext = setArgText('partlimit',(numpart,),False)
xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'],), True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
reversetext = '--reverse' if self.originalStackData.reverse else ''
defoctext = '--defocpair' if self.originalStackData.defocpair else ''
inverttext = '--no-invert' if not self.invert else ''
# Build the makestack2 command
cmd = "makestack2.py "
cmd += (" --single=%s --fromstackid=%d %s %s %s %s %s %s "%
(os.path.basename(newstackimagicfile),selectionid,stackid,lowpasstext,
highpasstext,partlimittext,reversetext,defoctext,inverttext))
cmd += (" --normalized %s --boxsize=%d --bin=%d "%
(xmipp_normtext,unbinnedboxsize,bin))
cmd += (" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "%
(stackpathname,stackid,projectid))
cmd += (" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d "%
(presetname,newstackrunname,newstackrundir,sessionname,sessionid))
cmd += (" --no-wait --no-commit --no-continue --jobtype=makestack2 ")
# Run the command
logfilepath = os.path.join(newstackrundir,'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd,logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Make sure our new stack params reflects the changes made
# Use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
self.stack['file'] = newstackroot+'.hed'
# Clean up
boxfiles = glob.glob("*.box")
for boxfile in boxfiles:
apFile.removeFile(boxfile)
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"):
"""
inputs:
aligned particle stack
number of classes
timestamp
output directory
output:
class averages
class variances
dendrogram.png
"""
if timestamp is None:
timestamp = apParam.makeTimestamp()
classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses))
classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses))
thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext)
### create class doc files
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
mySpider.toSpider(
"CL HE",
thresh,
spyder.fileFilter(dendrogramfile), # dendrogram doc file
rundir+"/classdoc_"+timestamp+"_****", # class doc file
)
### delete existing files
sys.stderr.write("delete existing files")
for dext in (".hed", ".img", dataext):
apFile.removeFile(classavg+dext)
apFile.removeFile(classvar+dext)
print ""
### create class averages
sys.stderr.write("create class averages")
for i in range(classes):
sys.stderr.write(".")
classnum = i+1
mySpider.toSpiderQuiet(
"AS R",
spyder.fileFilter(alignedstack)+"@******",
rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)),
"A",
(classavg+"@%04d" % (classnum)),
(classvar+"@%04d" % (classnum)),
)
mySpider.close()
print ""
### convert to IMAGIC
emancmd = "proc2d "+classavg+".spi "+classavg+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
emancmd = "proc2d "+classvar+".spi "+classvar+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return classavg,classvar
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"):
"""
inputs:
aligned particle stack
number of classes
timestamp
output directory
output:
class averages
class variances
dendrogram.png
"""
if timestamp is None:
timestamp = apParam.makeTimestamp()
classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses))
classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses))
thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext)
### create class doc files
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
mySpider.toSpider(
"CL HE",
thresh,
spyder.fileFilter(dendrogramfile), # dendrogram doc file
rundir+"/classdoc_"+timestamp+"_****", # class doc file
)
### delete existing files
sys.stderr.write("delete existing files")
for dext in (".hed", ".img", dataext):
apFile.removeFile(classavg+dext)
apFile.removeFile(classvar+dext)
print ""
### create class averages
sys.stderr.write("create %i class averages"%classes)
for i in range(classes):
sys.stderr.write(".")
classnum = i+1
mySpider.toSpiderQuiet(
"AS R",
spyder.fileFilter(alignedstack)+"@******",
rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)),
"A",
(classavg+"@%04d" % (classnum)),
(classvar+"@%04d" % (classnum)),
)
mySpider.close()
print ""
### convert to IMAGIC
emancmd = "proc2d "+classavg+".spi "+classavg+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
emancmd = "proc2d "+classvar+".spi "+classvar+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return classavg,classvar
def runAffinityPropagation(self, alignedstack):
### Get initial correlation values
### this is really, really slow
similarfile, simlist = self.fillSimilarityMatrix(alignedstack)
### Preference value stats
preffile = self.setPreferences(simlist)
### run apcluster.exe program
outfile = "clusters.out"
apDisplay.printMsg("Run apcluster.exe program")
apclusterexe = os.path.join(apParam.getAppionDirectory(), "bin/apcluster.exe")
apFile.removeFile(outfile)
clustercmd = apclusterexe+" "+similarfile+" "+preffile+" "+outfile
clusttime = time.time()
proc = subprocess.Popen(clustercmd, shell=True)
proc.wait()
apDisplay.printMsg("apCluster time: "+apDisplay.timeString(time.time()-clusttime))
if not os.path.isfile(outfile):
apDisplay.printError("apCluster did not run")
### Parse apcluster output file: clusters.out
apDisplay.printMsg("Parse apcluster output file: "+outfile)
clustf = open(outfile, "r")
### each line is the particle and the number is the class
partnum = 0
classes = {}
for line in clustf:
sline = line.strip()
if sline:
partnum += 1
classnum = int(sline)
if not classnum in classes:
classes[classnum] = [partnum,]
else:
classes[classnum].append(partnum)
clustf.close()
apDisplay.printMsg("Found %d classes"%(len(classes.keys())))
### Create class averages
classavgdata = []
classnames = classes.keys()
classnames.sort()
for classnum in classnames:
apDisplay.printMsg("Class %d, %d members"%(classnum, len(classes[classnum])))
#clsf = open('subcls%04d.lst'%(classnum), 'w')
#for partnum in classes[classnum]:
# clsf.write("%d\n"%(partnum))
#clsf.close()
classdatalist = apImagicFile.readParticleListFromStack(alignedstack, classes[classnum], msg=False)
classdatarray = numpy.asarray(classdatalist)
classavgarray = classdatarray.mean(0)
#mrc.write(classavgarray, 'subcls%04d.mrc'%(classnum))
classavgdata.append(classavgarray)
apFile.removeStack("classaverage-"+self.timestamp+".hed")
apImagicFile.writeImagic(classavgdata, "classaverage-"+self.timestamp+".hed")
return classes
def loopCleanUp(self, imgdata):
### last remove any existing boxed files, reset global params
shortname = apDisplay.short(imgdata['filename'])
shortfileroot = os.path.join(self.params['rundir'], shortname)
rmfiles = glob.glob(shortfileroot + "*")
if not self.params['keepall']:
for rmfile in rmfiles:
apFile.removeFile(rmfile)
def emanMrcToStack(partlist):
apFile.removeStack("emanmrc.hed", warn=False)
for part in partlist:
mrc.write(part, "temp.mrc")
emancmd = "proc2d temp.mrc emanmrc.hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apFile.removeFile("temp.mrc")
return
def emanSpiderToStack(partlist):
apFile.removeStack("emanspi.hed", warn=False)
for part in partlist:
spider.write(part, "temp.spi")
emancmd = "proc2d temp.spi emanspi.hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apFile.removeFile("temp.spi")
return
def initialBPRP(self, classnum, volfile, spiderstack, eulerfile, numpart,
pixrad):
# file that stores the number of iteration for BPRP
BPRPcount = os.path.join(self.params['rundir'], str(classnum),
"numiter.spi")
if (os.path.isfile(BPRPcount)):
apDisplay.printMsg("BP RP counter file exists: " + BPRPcount +
"! File will be deleted.")
apFile.removeFile(BPRPcount)
BPRPlambda = 2e-5
backproject.backprojectRP(spiderstack,
eulerfile,
volfile,
pixrad=pixrad,
classnum=classnum,
lambDa=BPRPlambda,
numpart=numpart)
count = 0
rounds = 0
### repeat BPRP for 100 times with different values of lambda or until BPRP manages to do 50 iterations
while count < 50 and rounds < 100:
if (os.path.isfile(BPRPcount)):
bc = open(BPRPcount, "r")
for line in bc.readlines():
value = line.split()
if value[0] == "1":
count = int(float(value[2]))
if count < 50:
apDisplay.printMsg(
"BPRP iteration is " + str(count) +
" (less than 50)... redoing BPRP")
bc.close()
apFile.removeFile(BPRPcount)
BPRPlambda = BPRPlambda / 2
backproject.backprojectRP(spiderstack,
eulerfile,
volfile,
pixrad=pixrad,
classnum=classnum,
lambDa=BPRPlambda,
numpart=numpart)
else:
apDisplay.printWarning("numiter is missing")
continue
rounds += 1
### print warning if BPRP reaches 100 rounds
if rounds >= 100:
apDisplay.printWarning(
"BPRP attempted 100 times but iteration is still less than 50. Check BPRP params."
)
return
