def renderSlice(density, box=None, tmpfile=None, sym='c1'):
"""
create mrc of central slice for viruses
"""
if isValidVolume(density) is False:
apDisplay.printError("Volume file is not valid")
if tmpfile is None:
tmpfile = density
if box is None:
boxdims = apFile.getBoxSize(tmpfile)
box = boxdims[0]
halfbox = int(box / 2)
tmphed = density + '.hed'
tmpimg = density + '.img'
hedcmd = ('proc3d %s %s' % (tmpfile, tmphed))
if sym.lower()[:4] != 'icos':
hedcmd = hedcmd + " rot=90"
proc = subprocess.Popen(hedcmd, shell=True)
proc.wait()
pngslice = density + '.slice.png'
slicecmd = ('proc2d %s %s first=%i last=%i' %
(tmphed, pngslice, halfbox, halfbox))
proc = subprocess.Popen(slicecmd, shell=True)
proc.wait()
apFile.removeStack(tmphed, warn=False)
return
def calcResolution(self, partlist, stackfile, apix):
### group particles by refnum
reflistsdict = {}
for partdict in partlist:
refnum = partdict['template']
partnum = partdict['num']
if not refnum in reflistsdict:
reflistsdict[refnum] = []
reflistsdict[refnum].append(partnum)
### get resolution
self.resdict = {}
boxsizetuple = apFile.getBoxSize(stackfile)
boxsize = boxsizetuple[0]
for refnum in reflistsdict.keys():
partlist = reflistsdict[refnum]
esttime = 3e-6 * len(partlist) * boxsize**2
apDisplay.printMsg("Ref num %d; %d parts; est time %s"
%(refnum, len(partlist), apDisplay.timeString(esttime)))
frcdata = apFourier.spectralSNRStack(stackfile, apix, partlist, msg=False)
frcfile = "frcplot-%03d.dat"%(refnum)
apFourier.writeFrcPlot(frcfile, frcdata, apix, boxsize)
res = apFourier.getResolution(frcdata, apix, boxsize)
self.resdict[refnum] = res
return
def start(self):
### load parameters
self.runparams = self.readRunParameters()
self.apix = apStack.getStackPixelSizeFromStackId(
self.runparams['stackid']) * self.runparams['bin']
self.Nlevels = len(
glob.glob("part" + self.params['timestamp'] + "_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part" + self.params[
'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
if self.runparams['align']:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels - 1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
if self.runparams['align']:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part" + self.params['timestamp'] +
"_level_%02d_.hed" % level, classnum + 1,
partdict[classnum], len(partdict))
def initValues(self, stackfile, numrequest=None):
### check for stack
if not os.path.isfile(stackfile):
apDisplay.printError("stackfile does not exist: "+stackfile)
### amount of free memory on machine (converted to bytes)
self.freememory = mem.free()*1024
self.message("Free memory: %s"%(apDisplay.bytes(self.freememory)))
### box size of particle
self.boxsize = apFile.getBoxSize(stackfile)[0]
self.message("Box size: %d"%(self.boxsize))
### amount of memory used per particles (4 bytes per pixel)
self.memperpart = self.boxsize**2 * 4.0
self.message("Memory used per part: %s"%(apDisplay.bytes(self.memperpart)))
### maximum number particles that fit into memory
self.maxpartinmem = self.freememory/self.memperpart
self.message("Max particles in memory: %d"%(self.maxpartinmem))
### number particles to fit into memory
self.partallowed = int(self.maxpartinmem/20.0)
self.message("Particles allowed in memory: %d"%(self.partallowed))
### number particles in stack
numpart = apFile.numImagesInStack(stackfile)
if self.numpart is None or self.numpart > numpart:
self.numpart = numpart
if numrequest is not None and self.numpart > numrequest:
self.numpart = numrequest
self.message("Number of particles in stack: %d"%(self.numpart))
if self.numpart > self.partallowed:
numchucks = math.ceil(self.numpart/float(self.partallowed))
self.stepsize = int(self.numpart/numchucks)
else:
numchucks = 1
self.stepsize = self.numpart
self.message("Particle loop num chunks: %d"%(numchucks))
self.message("Particle loop step size: %d"%(self.stepsize))
def start(self):
### copy stack to rundir
newstack = os.path.join(self.params['rundir'], "start.hed")
if os.path.isfile(newstack):
apDisplay.printError("Stack already exists")
emancmd = "proc2d %s %s" % (self.params['stackfile'], newstack)
if self.params['normalize'] is True:
emancmd += " edgenorm"
apEMAN.executeEmanCmd(emancmd)
### set final parameters
boxsize = apFile.getBoxSize(newstack)
apDisplay.printMsg("Boxsize: %i" % boxsize[0])
if not boxsize or boxsize <= 0:
apDisplay.printError("Could not determine stack size")
else:
self.boxsize = boxsize[0]
self.numpart = apFile.numImagesInStack(newstack)
apDisplay.printMsg("Num part: %i" % self.numpart)
if not self.numpart or self.numpart <= 0:
apDisplay.printError("Could not determine number of particles")
if self.numpart <= 1:
apDisplay.printError("Not enough particles to upload")
apStack.averageStack(newstack)
#self.params['commit'] = False
self.createStackData()
def renderSlice(density, box=None, tmpfile=None, sym='c1'):
"""
create mrc of central slice for viruses
"""
if isValidVolume(density) is False:
apDisplay.printError("Volume file is not valid")
if tmpfile is None:
tmpfile = density
if box is None:
boxdims = apFile.getBoxSize(tmpfile)
box = boxdims[0]
halfbox = int(box/2)
tmphed = density + '.hed'
tmpimg = density + '.img'
hedcmd = ('proc3d %s %s' % (tmpfile, tmphed))
if sym.lower()[:4] != 'icos':
hedcmd = hedcmd + " rot=90"
proc = subprocess.Popen(hedcmd, shell=True)
proc.wait()
pngslice = density + '.slice.png'
slicecmd = ('proc2d %s %s first=%i last=%i' % (tmphed, pngslice, halfbox, halfbox))
proc = subprocess.Popen(slicecmd, shell=True)
proc.wait()
apFile.removeStack(tmphed, warn=False)
return
def getParticlesPerCycle(self, stackfile):
"""
it more efficient to process X particles and write them to disk rather than
write each particle to disk each time.
particles are read using a memory map (numpy.memmap), so we can pretend to
continuously read all into memory
"""
### amount of free memory on machine (converted to bytes)
freememory = mem.free()*1024
self.message("Free memory: %s"%(apDisplay.bytes(freememory)))
### box size of particle
self.boxsize = apFile.getBoxSize(stackfile)[0]
self.message("Box size: %d"%(self.boxsize))
### amount of memory used per particles (4 bytes per pixel)
memperpart = self.boxsize**2 * 4.0
self.message("Memory used per part: %s"%(apDisplay.bytes(memperpart)))
### maximum number particles that fit into memory
maxpartinmem = freememory/memperpart
self.message("Max particles in memory: %d"%(maxpartinmem))
### number particles to fit into memory
partallowed = int(maxpartinmem/20.0)
self.message("Particles allowed in memory: %d"%(partallowed))
### number particles in stack
numpart = self.params['last']
if numpart > partallowed:
numcycles = math.ceil(numpart/float(partallowed))
stepsize = int(numpart/numcycles)
else:
numcycles = 1
stepsize = numpart
self.message("Particle loop num cycles: %d"%(numcycles))
self.message("Particle loop step size: %d"%(stepsize))
return stepsize
def start(self):
### copy stack to rundir
newstack = os.path.join(self.params['rundir'], "start.hed")
if os.path.isfile(newstack):
apDisplay.printError("Stack already exists")
emancmd = "proc2d %s %s"%(self.params['stackfile'], newstack)
if self.params['normalize'] is True:
emancmd += " edgenorm"
apEMAN.executeEmanCmd(emancmd)
### set final parameters
boxsize = apFile.getBoxSize(newstack)
apDisplay.printMsg("Boxsize: %i"%boxsize[0])
if not boxsize or boxsize <= 0:
apDisplay.printError("Could not determine stack size")
else:
self.boxsize = boxsize[0]
self.numpart = apFile.numImagesInStack(newstack)
apDisplay.printMsg("Num part: %i"%self.numpart)
if not self.numpart or self.numpart <= 0:
apDisplay.printError("Could not determine number of particles")
if self.numpart <= 1:
apDisplay.printError("Not enough particles to upload")
apStack.averageStack(newstack)
#self.params['commit'] = False
self.createStackData()
def scaleTemplates(self):
reffile = os.path.join(self.params['rundir'], "references.hed")
if self.params['apix'] != self.templatestack['apix']:
scalefactor = float(
self.templatestack['apix']) / self.params['apix']
templates = apImagicFile.readImagic(reffile)
scaledtemplates = []
for templatearray in templates['images']:
newarray = apTemplate.scaleTemplate(templatearray, scalefactor)
scaledtemplates.append(newarray)
apImagicFile.writeImagic(scaledtemplates, reffile)
refbox = apFile.getBoxSize(reffile)[0]
stbox = self.params['boxsize']
### now clip the references to get identical boxsizes
if stbox != refbox:
while os.path.isfile(reffile + ".new.img"):
apFile.removeStack(reffile + ".new.img")
emancmd = "proc2d " + reffile + " " + reffile + ".new.hed clip=" + str(
stbox) + " edgenorm"
apParam.runCmd(emancmd, "EMAN")
os.rename(reffile + ".new.hed", reffile)
os.rename(reffile + ".new.img", reffile[:-4] + ".img")
return
def uploadDensity(self, volfile):
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq['path'] = appiondata.ApPathData(
path=os.path.dirname(os.path.abspath(volfile)))
densq['name'] = os.path.basename(volfile)
densq['hidden'] = False
densq['norm'] = True
densq['symmetry'] = self.params['symdata']
#densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25)
densq['pixelsize'] = self.apix
densq['mass'] = self.mass
densq['boxsize'] = apFile.getBoxSize(volfile)[0]
densq['lowpass'] = self.params['lowpass']
#densq['highpass'] = self.params['highpasspart']
#densq['mask'] = self.params['radius']
densq['description'] = "EMDB id %d density" % (self.params['emdbid'])
if self.mass is not None:
densq['description'] += " with mass of %d kDa" % (self.mass)
densq['resolution'] = self.params['lowpass']
densq['session'] = self.sessiondata
densq['md5sum'] = apFile.md5sumfile(volfile)
densq['emdbid'] = self.params['emdbid']
if self.params['commit'] is True:
densq.insert()
return
def uploadDensity(self, volfile): ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = self.params['symdata'] #densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25) densq['pixelsize'] = self.apix densq['mass'] = self.mass densq['boxsize'] = apFile.getBoxSize(volfile)[0] densq['lowpass'] = self.params['lowpass'] #densq['highpass'] = self.params['highpasspart'] #densq['mask'] = self.params['radius'] densq['description'] = "EMDB id %d density"%(self.params['emdbid']) if self.mass is not None: densq['description'] += " with mass of %d kDa"%(self.mass) densq['resolution'] = self.params['lowpass'] densq['session'] = self.sessiondata densq['md5sum'] = apFile.md5sumfile(volfile) densq['emdbid'] = self.params['emdbid'] if self.params['commit'] is True: densq.insert() return
def calcResolution(self, partlist, stackfile, apix):
### group particles by refnum
reflistsdict = {}
for partdict in partlist:
refnum = partdict['template']
partnum = partdict['num']
if not refnum in reflistsdict:
reflistsdict[refnum] = []
reflistsdict[refnum].append(partnum)
### get resolution
self.resdict = {}
boxsizetuple = apFile.getBoxSize(stackfile)
boxsize = boxsizetuple[0]
for refnum in reflistsdict.keys():
partlist = reflistsdict[refnum]
esttime = 3e-6 * len(partlist) * boxsize**2
apDisplay.printMsg("Ref num %d; %d parts; est time %s"
%(refnum, len(partlist), apDisplay.timeString(esttime)))
frcdata = apFourier.spectralSNRStack(stackfile, apix, partlist, msg=False)
frcfile = "frcplot-%03d.dat"%(refnum)
apFourier.writeFrcPlot(frcfile, frcdata, apix, boxsize)
res = apFourier.getResolution(frcdata, apix, boxsize)
self.resdict[refnum] = res
return
def initValues(self, stackfile, numrequest=None):
### check for stack
if not os.path.isfile(stackfile):
apDisplay.printError("stackfile does not exist: " + stackfile)
### amount of free memory on machine (converted to bytes)
self.freememory = mem.free() * 1024
self.message("Free memory: %s" % (apDisplay.bytes(self.freememory)))
### box size of particle
self.boxsize = apFile.getBoxSize(stackfile)[0]
self.message("Box size: %d" % (self.boxsize))
### amount of memory used per particles (4 bytes per pixel)
self.memperpart = self.boxsize**2 * 4.0
self.message("Memory used per part: %s" %
(apDisplay.bytes(self.memperpart)))
### maximum number particles that fit into memory
self.maxpartinmem = self.freememory / self.memperpart
self.message("Max particles in memory: %d" % (self.maxpartinmem))
### number particles to fit into memory
self.partallowed = int(self.maxpartinmem / 20.0)
self.message("Particles allowed in memory: %d" % (self.partallowed))
### number particles in stack
numpart = apFile.numImagesInStack(stackfile)
if self.numpart is None or self.numpart > numpart:
self.numpart = numpart
if numrequest is not None and self.numpart > numrequest:
self.numpart = numrequest
self.message("Number of particles in stack: %d" % (self.numpart))
if self.numpart > self.partallowed:
numchucks = math.ceil(self.numpart / float(self.partallowed))
self.stepsize = int(self.numpart / numchucks)
else:
numchucks = 1
self.stepsize = self.numpart
self.message("Particle loop num chunks: %d" % (numchucks))
self.message("Particle loop step size: %d" % (self.stepsize))
def filterAndChimera(density, res=30, apix=None, box=None, chimtype='snapshot',
contour=None, zoom=1.0, sym='c1', color=None, silhouette=True, mass=None):
"""
filter volume and then create a few snapshots for viewing on the web
"""
if isValidVolume(density) is False:
apDisplay.printError("Volume file %s is not valid"%(density))
if box is None:
boxdims = apFile.getBoxSize(density)
box = boxdims[0]
### if eotest failed, filter to 30
if not res or str(res) == 'nan':
res = 30
### low pass filter the volume to 60% of reported res
tmpf = os.path.abspath(density+'.tmp.mrc')
density = os.path.abspath(density)
filtres = 0.6*res
shrinkby = 1
if box is not None and box > 250:
shrinkby = int(math.ceil(box/160.0))
if box % (2*shrinkby) == 0:
### box is divisible by shrink by
lpcmd = ('proc3d %s %s apix=%.3f tlp=%.2f shrink=%d origin=0,0,0 norm=0,1'
% (density, tmpf, apix, filtres, shrinkby))
else:
### box not divisible by shrink by, need a clip
clip = math.floor(box/shrinkby/2.0)*2*shrinkby
lpcmd = ('proc3d %s %s apix=%.3f tlp=%.2f shrink=%d origin=0,0,0 norm=0,1 clip=%d,%d,%d'
% (density, tmpf, apix, filtres, shrinkby, clip, clip, clip))
else:
lpcmd = ('proc3d %s %s apix=%.3f tlp=%.2f origin=0,0,0 norm=0,1'
% (density, tmpf, apix, filtres))
apDisplay.printMsg("Low pass filtering model for images")
proc = subprocess.Popen(lpcmd, shell=True)
proc.wait()
### flatten solvent
vol = mrc.read(tmpf)
numpy.where(vol < 0, 0.0, vol)
mrc.write(vol, tmpf)
del vol
### contour volume to mass
if mass is not None:
setVolumeMass(tmpf, apix*shrinkby, mass)
contour = 1.0
### set pixelsize and origin
recmd = "proc3d %s %s apix=%.3f origin=0,0,0"%(tmpf, tmpf, apix)
proc = subprocess.Popen(recmd, shell=True)
proc.wait()
### render images
renderSlice(density, box=box, tmpfile=tmpf, sym=sym)
if chimtype != 'snapshot':
renderAnimation(tmpf, contour, zoom, sym, color, silhouette, name=density)
elif chimtype != 'animate':
renderSnapshots(tmpf, contour, zoom, sym, color, silhouette, name=density)
apFile.removeFile(tmpf)
def insertRunIntoDatabase(self, alignedPartStack, alignedClassStack, runparams):
apDisplay.printMsg("Inserting ISAC Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = runparams['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup ISAC like run
isacq = appiondata.ApSparxISACRunData()
isacq['runname'] = runparams['runname']
isacq['job'] = self.getISACJobData(runparams)
### finish alignment run
alignrunq['isacrun'] = isacq
alignrunq['hidden'] = False
alignrunq['runname'] = runparams['runname']
alignrunq['description'] = runparams['description']
alignrunq['lp_filt'] = runparams['lowpass']
alignrunq['hp_filt'] = runparams['highpass']
alignrunq['bin'] = runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = alignedPartStack
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = alignedClassStack
alignstackq['iteration'] = self.lastiter
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: "+alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(runparams['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])*self.params['bin']
alignstackq['description'] = runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.numPart
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def scaleTemplates(self): reffile = os.path.join(self.params['rundir'], "references.hed") if self.params['apix'] != self.templatestack['apix']: scalefactor = float(self.templatestack['apix']) / self.params['apix'] templates = apImagicFile.readImagic(reffile) scaledtemplates = [] for templatearray in templates['images']: newarray = apTemplate.scaleTemplate(templatearray, scalefactor) scaledtemplates.append(newarray) apImagicFile.writeImagic(scaledtemplates, reffile) refbox = apFile.getBoxSize(reffile)[0] stbox = self.params['boxsize'] ### now clip the references to get identical boxsizes if stbox != refbox: while os.path.isfile(reffile+".new.img"): apFile.removeStack(reffile+".new.img") emancmd = "proc2d "+reffile+" "+reffile+".new.hed clip="+str(stbox)+" edgenorm" apParam.runCmd(emancmd, "EMAN") os.rename(reffile+".new.hed", reffile) os.rename(reffile+".new.img", reffile[:-4]+".img") return
def start(self):
### load parameters
self.runparams = self.readRunParameters()
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
if self.runparams['align']:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
if self.runparams['align']:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
def checkConflicts(self):
""" basic error handlign for running scripts """
### check for IMAGIC installation
try:
self.imagicroot = apIMAGIC.checkImagicExecutablePath()
except:
c = "some IMAGIC functions, e.g. prealignment and angular reconstitution "
c+= "may not work. Use EMAN instead"
apDisplay.printWarning(c)
self.imagicroot = None
self.params['useEMAN1'] is True
### check class averages
if self.params['templatestackid'] is not None:
self.stackdata = appiondata.ApTemplateStackData.direct_query(self.params['templatestackid'])
self.clsname = self.stackdata['templatename']
self.params['apix'] = self.stackdata['apix']
self.params['boxsize'] = self.stackdata['boxsize']
self.params['oldavgs'] = os.path.join(self.stackdata['path']['path'], self.clsname[:-4]+".hed")
self.params['avgs'] = os.path.basename(self.clsname)
elif self.params['clusterid'] is not None:
self.stackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
self.clsname = self.stackdata['avg_imagicfile']
self.params['apix'] = self.stackdata['clusterrun']['pixelsize']
self.params['boxsize'] = self.stackdata['clusterrun']['boxsize']
self.params['oldavgs'] = os.path.join(self.stackdata['path']['path'], self.clsname[:-4]+".hed")
self.params['avgs'] = self.clsname
elif self.params['classavgs'] is not None:
if self.params['apix'] is None:
apDisplay.printError("enter pixel size for manually uploaded classes")
self.params['oldavgs'] = self.params['classavgs']
bavgs = os.path.basename(self.params['classavgs'])
self.params['avgs'] = bavgs
else:
apDisplay.printError("enter class averages for the run")
if not os.path.isfile(os.path.abspath(self.params['oldavgs'])):
apDisplay.printError("cannot find input class averages")
if self.params['templatestackid'] is not None and self.params['clusterid'] is not None:
apDisplay.printError("enter either templatestack ID OR cluster ID")
if self.params['templatestackid'] is not None and self.params['classavgs'] is not None:
apDisplay.printError("enter either templatestack ID OR manually uploaded classes")
if self.params['clusterid'] is not None and self.params['classavgs'] is not None:
apDisplay.printError("enter either cluster ID OR manually uploaded classes")
### pretreatment
if self.imagicroot is not None and self.params['prealign'] is True:
warning = "particles will not be prealigned, IMAGIC is not installed or path "
warning+= "to IMAGIC cannot be located"
apDisplay.printWarning(warning)
### basic input parameters
self.params['numpart'] = apFile.numImagesInStack(self.params['oldavgs'])
if self.params['numpart'] < 3:
apDisplay.printError("need at least 3 class averages in stack to run")
self.params['boxsize'] = apFile.getBoxSize(self.params['oldavgs'])[0]
if self.params['num_volumes'] is None:
apDisplay.printError("specify the number of volumes to produce")
if self.params['apix'] is None:
apDisplay.printError("enter pixel size of class averages")
### refinement class averages
if self.params['r_templatestackid'] is not None:
self.rstackdata = appiondata.ApTemplateStackData.direct_query(self.params['r_templatestackid'])
self.rclsname = self.rstackdata['templatename']
self.params['refineapix'] = self.rstackdata['apix']
self.params['refineboxsize'] = self.rstackdata['boxsize']
self.params['oldravgs'] = os.path.join(self.rstackdata['path']['path'], self.rclsname[:-4]+".hed")
self.params['ravgs'] = os.path.basename(self.rclsname)
elif self.params['r_clusterid'] is not None:
self.rstackdata = appiondata.ApClusteringStackData.direct_query(self.params['r_clusterid'])
self.rclsname = self.rstackdata['avg_imagicfile']
self.params['refineapix'] = self.rstackdata['clusterrun']['pixelsize']
self.params['refineboxsize'] = self.rstackdata['clusterrun']['boxsize']
self.params['oldravgs'] = os.path.join(self.rstackdata['path']['path'], self.rclsname[:-4]+".hed")
self.params['ravgs'] = self.rclsname
elif self.params['refine_classavgs'] is not None:
rbavgs = os.path.basename(self.params['refine_classavgs'])
self.params['ravgs'] = rbavgs
self.params['refineboxsize'] = apFile.getBoxSize(self.params['oldravgs'])[0]
if self.params['refineapix'] is None:
warning = "refinement pixel size not specified ... assuming it is the same"
apDisplay.printWarning(warning)
self.params['refineapix'] = self.params['apix']
else:
# self.params['ravgs'] = None
self.params['oldravgs'] = self.params['avgs']
self.params['ravgs'] = self.params['avgs']
self.params['refineapix'] = self.params['apix']
self.params['refineboxsize'] = self.params['boxsize']
if not os.path.isfile(os.path.abspath(self.params['oldravgs'])):
warning = "cannot find class averages for refinement using the "
warning+= "specified path. Original averages will be used instead."
apDisplay.printWarning(warning)
self.params['oldravgs'] = self.params['avgs']
self.params['ravgs'] = self.params['avgs']
### check for scaling
if self.params['scale'] is True:
self.scalefactor = float(64.0 / self.params['boxsize'])
self.params['apix'] = self.params['apix'] / self.scalefactor
self.params['boxsize'] = 64
else:
self.scalefactor = 1
### angular reconstitution checks
if self.params['keep_ordered'] < 1.0: ### probably specified as a fraction
self.params['keep_ordered'] = self.params['keep_ordered'] * 100 ### convert to percentage
self.params['keep_ordered_num'] = self.params['numpart'] * self.params['keep_ordered'] / 100
### EMAN1 cross common lines checks
if self.params['useEMAN1'] is True:
if self.params['images_per_volume'] is None:
self.params['images_per_volume'] = int(self.params['numpart'] * 0.66)
if self.params['images_per_volume'] > self.params['numpart']:
warning = "number of images per volume greater than number of class averages "
warning+= "in input. Setting to (0.66)*(num_averages)"
apDisplay.printWarning(warning)
self.params['images_per_volume'] = int(self.params['numpart'] * 0.66)
if self.params['images_per_volume'] > self.params['numpart'] * 0.75:
warning = "consider asking for less images per volume to provide more variety "
warning = "during common lines search"
apDisplay.printWarning(warning)
### number of processors for threading ONLY works on a single node
self.params['threadnproc'] = apParam.getNumProcessors()
### refinement parameters
if self.params['linmask'] == 0:
self.params['linmask'] = (self.params['boxsize']-2)/2*self.params['apix']
if self.params['mask_radius'] is None and self.params['linmask'] != 0:
self.params['mask_radius'] = self.params['linmask'] * 1.2
elif self.params['mask_radius'] is None and self.params['linmask'] == 0:
self.params['mask_radius'] = self.params['boxsize'] * self.params['refineapix']
if self.params['outer_radius'] is None:
self.params['outer_radius'] = self.params['mask_radius'] * 0.8
return
def alignStack(oldstack, alignedstack, partlist, dataext=".spi"):
"""
write aligned stack -- with python loop
inputs:
oldstack
newstack (empty)
list of particle dictionaries for operations
modifies:
newstack
output:
none
I tried this loop in both spider and python;
python was faster?!? -neil
"""
if not os.path.isfile(oldstack + dataext):
apDisplay.printError("Could not find original stack: " + oldstack +
dataext)
boxsize = apFile.getBoxSize(oldstack + dataext)
apDisplay.printMsg("applying alignment parameters to stack")
apFile.removeFile(alignedstack + dataext)
count = 0
t0 = time.time()
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext,
logo=True,
nproc=nproc,
log=False)
#create stack in core
numpart = len(partlist)
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d" % (boxsize), #boxsize
str(numpart + 1), #num part to create in memory
str(numpart + 1), #max particle number
)
for partdict in partlist:
partnum = partdict['num']
#if partdict['num'] in [3,6,7]:
# print partdict['num'], partdict['template'], partdict['mirror'], round(partdict['rot'],3)
### Rotate and Shift operations
count += 1
#rotate/shift
mySpider.toSpiderQuiet(
"RT SQ",
spyder.fileFilter(oldstack) + "@" + ("%06d" % (partnum)),
"_1",
str(partdict['rot']),
str(partdict['xshift']) + "," + str(partdict['yshift']),
)
#mirror, if necessary
if 'mirror' in partdict and partdict['mirror'] is True:
mySpider.toSpiderQuiet(
"MR",
"_1",
"_2@" + ("%06d" % (partnum)),
"Y",
)
else:
mySpider.toSpiderQuiet(
"CP",
"_1",
"_2@" + ("%06d" % (partnum)),
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP",
"_2@",
spyder.fileFilter(alignedstack) + "@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE",
"_2",
)
mySpider.close()
apDisplay.printMsg("Completed transforming %d particles in %s" %
(count, apDisplay.timeString(time.time() - t0)))
if count < 1:
apDisplay.printError("Failed to transform any particles")
if not os.path.isfile(alignedstack + dataext):
apDisplay.printError("Failed to create stack " + alignedstack +
dataext)
return
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"):
"""
takes a selfile and creates an EMAN stack
"""
selfile = os.path.abspath(selfile)
stackfile = os.path.abspath(stackfile)
if stackfile[-4:] != ".hed":
apDisplay.printWarning("Stack file does not end in .hed")
stackfile = stackfile[:-4] + ".hed"
apDisplay.printColor("Merging files into a stack, this can take a while",
"cyan")
starttime = time.time()
if not os.path.isfile(selfile):
apDisplay.printError("selfile does not exist: " + selfile)
### Process selfile
fh = open(selfile, 'r')
filelist = []
for line in fh:
sline = line.strip()
if sline:
args = sline.split()
if (len(args) > 1):
filename = args[0].strip()
filelist.append(filename)
fh.close()
### Set variables
boxsize = apFile.getBoxSize(filelist[0])
partperiter = int(1e9 / (boxsize[0]**2) / 16.)
if partperiter > 4096:
partperiter = 4096
apDisplay.printMsg("Using %d particle per iteration" % (partperiter))
numpart = len(filelist)
if numpart < partperiter:
partperiter = numpart
### Process images
imgnum = 0
stacklist = []
stackroot = stackfile[:-4]
### get memory in kB
startmem = mem.active()
while imgnum < len(filelist):
filename = filelist[imgnum]
index = imgnum % partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
#sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index))
if mem.active() - startmem > 2e6:
apDisplay.printWarning("Out of memory")
if index < 1:
#print "img num", imgnum
### deal with large stacks, reset loop
if imgnum > 0:
sys.stderr.write("\n")
stackname = "%s-%d.hed" % (stackroot, imgnum)
apDisplay.printMsg("writing single particles to file " +
stackname)
stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
perpart = (time.time() - starttime) / imgnum
apDisplay.printColor(
"part %d of %d :: %.1fM mem :: %s/part :: %s remain" %
(imgnum + 1, numpart, (mem.active() - startmem) / 1024.,
apDisplay.timeString(perpart),
apDisplay.timeString(perpart * (numpart - imgnum))),
"blue")
stackarray = []
### merge particles
if filetype == "mrc":
partimg = mrc.read(filename)
else:
partimg = spider.read(filename)
stackarray.append(partimg)
imgnum += 1
### write remaining particles to file
sys.stderr.write("\n")
stackname = "%s-%d.hed" % (stackroot, imgnum)
apDisplay.printMsg("writing particles to file " + stackname)
stacklist.append(stackname)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### merge stacks
apFile.removeStack(stackfile, warn=False)
apImagicFile.mergeStacks(stacklist, stackfile)
print stackfile
filepart = apFile.numImagesInStack(stackfile)
if filepart != numpart:
apDisplay.printError(
"number merged particles (%d) not equal number expected particles (%d)"
% (filepart, numpart))
for stackname in stacklist:
apFile.removeStack(stackname, warn=False)
### summarize
apDisplay.printColor(
"merged %d particles in %s" %
(imgnum, apDisplay.timeString(time.time() - starttime)), "cyan")
def crossCorrelateAndShift(infile, reffile, alignfile, ccdocfile, numpart, dataext=".spi"):
### rewriten to do the whole thing in memory in SPIDER, it should be faster
starttime = time.time()
infile = spyder.fileFilter(infile)
reffile = spyder.fileFilter(reffile)
alignfile = spyder.fileFilter(alignfile)
partimg = "_4"
ccmap = "_5"
windccmap = "_6"
boxsize = apFile.getBoxSize(infile+dataext)
if not os.path.isfile(infile+dataext):
apDisplay.printError("input stack file not found: "+infile+dataext)
if not os.path.isfile(reffile+dataext):
apDisplay.printError("reference stack file not found: "+reffile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
### Allocate empty stack
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d"%(boxsize), #boxsize
str(numpart+1), #num part to create in memory
str(numpart+1), #max particle number
)
partnum = 0
while partnum < numpart:
partnum+=1
mySpider.toSpiderQuiet("CP",
infile+("@%05d"%(partnum)), #picture
partimg,
)
### cross correlate images; reversed order to avoid -1*shift
mySpider.toSpiderQuiet("CC N",
reffile+("@%05d"%(partnum)), #reference
partimg, #picture
ccmap, #output file
)
### cannot shift more the 1/4 size of the image
mySpider.toSpiderQuiet("FI x52", partimg, "12" )
mySpider.toSpiderQuiet("x54=int(x52/2)") #window size
mySpider.toSpiderQuiet("x55=int(x52/4)") #window topleft
mySpider.toSpiderQuiet("WI",
ccmap, #input file
windccmap, #output file
"x54,x54", #window size
"x55,x55", #window origin
)
### find the cross-correlation peak
mySpider.toSpiderQuiet("x56=int(x52/4)+1") #center of window
mySpider.toSpiderQuiet("PK M x11,x12,x13,x14",
windccmap, #input ccmap file
"x56,x56", #origin coordinates
)
### save info to doc file
mySpider.toSpiderQuiet("SD %d,x13,x14"%(partnum),
ccdocfile, #input ccmap file
)
### shift the images images
mySpider.toSpiderQuiet("SH",
partimg, #old stack
("_2@%05d"%(partnum)), #new stack
"x13,x14", #shift value file
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP", "_2@",
alignfile+"@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE", "_2",
)
mySpider.close()
apDisplay.printColor("finished shifting particles in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def commitToDatabase(self):
"""
insert the results into the database
"""
### expected result for an alignment run:
### 1. aligned particle stack in IMAGIC
### 2. rotation, shift, and quality parameters for each particle
### 3. which particles belongs to which class
### 4. stack file with the class averages
alignedstack = os.path.join(self.params['rundir'], "ptcl.hed")
refstack = os.path.join(self.params['rundir'], "iter.final.hed")
averagemrc = os.path.join(self.params['rundir'], "average.mrc")
apStack.averageStack(alignedstack, averagemrc)
particlemapping = self.determineClassOwnership()
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError(
"Run name '" + runparams['runname'] +
"' and path already exisclassmappingt in database")
### setup eman refine2d run
emanrefinetwodq = appiondata.ApEMANRefine2dRunData()
emanrefinetwodq['runname'] = self.params['runname']
emanrefinetwodq['run_seconds'] = time.time() - self.t0
emanrefinetwodq['num_iters'] = self.params['numiter']
emanrefinetwodq['num_classes'] = self.params['numclasses']
### finish alignment run
alignrunq['refine2drun'] = emanrefinetwodq
alignrunq['hidden'] = False
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['bin'] = self.params['bin']
### setup alignment stackalignimagicfile
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = os.path.basename(alignedstack)
alignstackq['avgmrcfile'] = os.path.basename(averagemrc)
alignstackq['refstackfile'] = os.path.basename(refstack)
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: " +
alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printErrrefqor("could not find reference stack file: " +
refstackfile)
### continue setting values
alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid']) * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = apFile.numImagesInStack(
alignimagicfilepath)
### inserting particles and references
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for emanpartnum in range(self.params['numpart']):
partnum = emanpartnum + 1
if partnum % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refnum = particlemapping[emanpartnum]
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### TODO: create mrc file
#refq['mrcfile'] = refbase+".mrc"
#reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
#if not os.path.isfile(reffile):
# emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc"
# apEMAN.executeEmanCmd(emancmd, verbose=False)
#if not os.path.isfile(reffile):
# apDisplay.printError("could not find reference file: "+reffile)
### TODO: get resolution
#refq['ssnr_resolution'] = TODO
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partnum)
alignpartq['stackpart'] = stackpartdata
### TODO: get the alignment parameters
#alignpartq['xshift'] = partdict['xshift']
#alignpartq['yshift'] = partdict['yshift']
#alignpartq['rotation'] = partdict['inplane']
#alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
### TODO: get the score
#alignpartq['score'] = partdict['score']
### insert
if self.params['commit'] is True:
alignpartq.insert()
return
def start(self):
### get stack parameteres
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackId'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackId'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackId'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### copy stack into working directory
if os.path.isfile(self.stack['file']):
apDisplay.printColor("copying stack into running directoroy",
"cyan")
if self.stack['file'][-4:] == ".img" or self.stack['file'][
-4:] == ".hed":
strippedfile = self.stack['file'][:-4]
else:
strippedfile = self.stack['file']
while os.path.isfile(
os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(
os.path.join(self.params['rundir'], "start.img"))
emancmd = "proc2d "+strippedfile+".hed "+os.path.join(self.params['rundir'], "start.hed ")+\
"first=0 last="+str(self.params['numpart']-1)
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("stack not found in database")
### get template stack parameters
self.templatestack = {}
self.templatestack[
'data'] = appiondata.ApTemplateStackData.direct_query(
self.params['templateStackId'])
self.templatestack['apix'] = self.templatestack['data']['apix']
self.templatestack['boxsize'] = self.templatestack['data']['boxsize']
self.templatestack['file'] = os.path.join(
self.templatestack['data']['path']['path'],
self.templatestack['data']['templatename'])
self.templatestack['numimages'] = self.templatestack['data'][
'numimages']
### copy templates into working directory
if os.path.isfile(self.templatestack['file']):
apDisplay.printColor("copying templates into running directoroy",
"cyan")
ts = os.path.join(self.params['rundir'], "references.img")
while os.path.isfile(ts):
apFile.removeStack(ts)
if self.templatestack['file'][-4:] == ".img" or self.templatestack[
'file'][-4:] == ".hed":
strippedfile = self.templatestack['file'][:-4]
else:
strippedfile = self.templatestack['file']
emancmd = "proc2d " + strippedfile + ".img " + ts
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("template stack not found in database")
### set new pixelsize
if self.params['bin'] is not None and self.params['bin'] != 0:
self.params['apix'] = float(self.stack['apix']) * int(
self.params['bin'])
else:
self.params['apix'] = self.stack['apix']
### scale, low-pass, and high-pass filter stack ... do this with imagic, because it determines the appropriate boxsizes
scalingbatchfile = self.createImagicBatchFileScaling()
preptime = time.time()
proc = subprocess.Popen("chmod 775 " + str(scalingbatchfile),
shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(scalingbatchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(
os.path.join(self.params['rundir'], "prepareStack.log"))
apDisplay.printColor(
"finished IMAGIC in " +
apDisplay.timeString(time.time() - preptime), "cyan")
### set new boxsize, done only after scaling is complete
if self.params['bin'] is not None:
self.params['boxsize'] = apFile.getBoxSize(
os.path.join(self.params['rundir'], "start.hed"))[0]
else:
self.params['boxsize'] = self.stack['boxsize']
### make sure template stack boxsize matches that of the input stack
if self.params['apix'] != self.templatestack['apix'] or self.params[
'boxsize'] != self.templatestack['boxsize']:
self.scaleTemplates()
starttime = time.time()
print self.params
print "... stack pixel size: " + str(self.params['apix'])
print "... stack box size: " + str(self.params['boxsize'])
apDisplay.printColor(
"Running IMAGIC .batch file: See multiReferenceAlignment.log file for details",
"cyan")
### create IMAGIC batch file
batchfile = self.createImagicBatchFileMRA()
### execute IMAGIC batch file
aligntime0 = time.time()
proc = subprocess.Popen("chmod 775 " + str(batchfile), shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(batchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(
os.path.join(self.params['rundir'], "multiReferenceAlignment.log"))
apDisplay.printColor(
"finished IMAGIC in " +
apDisplay.timeString(time.time() - aligntime0), "cyan")
### get particle parameters (shift, rotate, refnum, mirror, ccc)
partparams = self.getParticleParams()
### average stack
alignstack = os.path.join(self.params['rundir'], "alignstack.hed")
apStack.averageStack(alignstack)
### normalize particles (otherwise found problems in viewing with stackviewer)
emancmd = "proc2d " + alignstack + " " + alignstack + ".norm.hed norm"
while os.path.isfile(alignstack + ".norm.img"):
apFile.removeStack(alignstack + ".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(alignstack + ".norm.hed", alignstack)
os.rename(alignstack + ".norm.img", alignstack[:-4] + ".img")
### normalize references
emancmd = "proc2d " + ts + " " + ts + ".norm.hed norm"
while os.path.isfile(ts + ".norm.img"):
apFile.removeStack(ts + ".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(ts + ".norm.hed", ts)
os.rename(ts + ".norm.img", ts[:-4] + ".img")
### remove copied stack
while os.path.isfile(os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(os.path.join(self.params['rundir'],
"start.img"))
### insert run into database
self.insertAlignmentRun(insert=True)
self.insertParticlesIntoDatabase(partparams, insert=True)
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### setup Xmipp command
aligntime = time.time()
xmippopts = (" -i "+os.path.join(self.params['rundir'], self.params['localstack'])
+" --nref "+str(self.params['numrefs'])
+" --iter "+str(self.params['maxiter'])
+" --odir "+str(self.params['rundir'])
+" --oroot "+ "part"+str(self.params['timestamp'])
+" --classifyAllImages"
)
if self.params['correlation']:
xmippopts += " --distance correlation"
if self.params['classical']:
xmippopts += " --classicalMultiref"
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath(self.execFile, die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp 3", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### post-processing
# Create a stack for the class averages at each level
Nlevels=glob.glob("level_*")
for level in Nlevels:
digits = level.split("_")[1]
apParam.runCmd("xmipp_image_convert -i "+level+"/part"+self.params['timestamp']+"*xmd -o part"
+self.params['timestamp']+"_level_"+digits+"_.hed", package="Xmipp 3", verbose=True)
if self.params['align']:
apParam.runCmd("xmipp_transform_geometry -i images.xmd -o %s_aligned.stk --apply_transform" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apParam.runCmd("xmipp_image_convert -i %s_aligned.xmd -o alignedStack.hed" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apFile.removeFile("%s_aligned.xmd" % self.params['timestamp'])
apFile.removeFile("%s_aligned.stk" % self.params['timestamp'])
self.parseOutput()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
self.clearIntermediateFiles()
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -codes "+str(self.params['numrefs'])
+" -iter "+str(self.params['maxiter'])
+" -o "+os.path.join(self.params['rundir'], "part"+self.params['timestamp'])
)
if self.params['fast']:
xmippopts += " -fast "
if self.params['correlation']:
xmippopts += " -useCorrelation "
if self.params['classical']:
xmippopts += " -classicalMultiref "
if self.params['align']:
xmippopts += " -alignImages "
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### minor post-processing
self.createReferenceStack()
self.parseOutput()
self.clearIntermediateFiles()
# self.readyUploadFlag()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
def commitToDatabase(self):
"""
insert the results into the database
"""
### expected result for an alignment run:
### 1. aligned particle stack in IMAGIC
### 2. rotation, shift, and quality parameters for each particle
### 3. which particles belongs to which class
### 4. stack file with the class averages
alignedstack = os.path.join(self.params['rundir'], "ptcl.hed")
refstack = os.path.join(self.params['rundir'], "iter.final.hed")
averagemrc = os.path.join(self.params['rundir'], "average.mrc")
apStack.averageStack(alignedstack, averagemrc)
particlemapping = self.determineClassOwnership()
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']
+"' and path already exisclassmappingt in database")
### setup eman refine2d run
emanrefinetwodq = appiondata.ApEMANRefine2dRunData()
emanrefinetwodq['runname'] = self.params['runname']
emanrefinetwodq['run_seconds'] = time.time() - self.t0
emanrefinetwodq['num_iters'] = self.params['numiter']
emanrefinetwodq['num_classes'] = self.params['numclasses']
### finish alignment run
alignrunq['refine2drun'] = emanrefinetwodq
alignrunq['hidden'] = False
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['bin'] = self.params['bin']
### setup alignment stackalignimagicfile
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = os.path.basename(alignedstack)
alignstackq['avgmrcfile'] = os.path.basename(averagemrc)
alignstackq['refstackfile'] = os.path.basename(refstack)
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: "+alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printErrrefqor("could not find reference stack file: "+refstackfile)
### continue setting values
alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])*self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = apFile.numImagesInStack(alignimagicfilepath)
### inserting particles and references
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for emanpartnum in range(self.params['numpart']):
partnum = emanpartnum+1
if partnum % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refnum = particlemapping[emanpartnum]
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### TODO: create mrc file
#refq['mrcfile'] = refbase+".mrc"
#reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
#if not os.path.isfile(reffile):
# emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc"
# apEMAN.executeEmanCmd(emancmd, verbose=False)
#if not os.path.isfile(reffile):
# apDisplay.printError("could not find reference file: "+reffile)
### TODO: get resolution
#refq['ssnr_resolution'] = TODO
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'], partnum)
alignpartq['stackpart'] = stackpartdata
### TODO: get the alignment parameters
#alignpartq['xshift'] = partdict['xshift']
#alignpartq['yshift'] = partdict['yshift']
#alignpartq['rotation'] = partdict['inplane']
#alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
### TODO: get the score
#alignpartq['score'] = partdict['score']
### insert
if self.params['commit'] is True:
alignpartq.insert()
return
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"):
"""
takes a selfile and creates an EMAN stack
"""
selfile = os.path.abspath(selfile)
stackfile = os.path.abspath(stackfile)
if stackfile[-4:] != ".hed":
apDisplay.printWarning("Stack file does not end in .hed")
stackfile = stackfile[:-4]+".hed"
apDisplay.printColor("Merging files into a stack, this can take a while", "cyan")
starttime = time.time()
if not os.path.isfile(selfile):
apDisplay.printError("selfile does not exist: "+selfile)
### Process selfile
fh = open(selfile, 'r')
filelist = []
for line in fh:
sline = line.strip()
if sline:
args=sline.split()
if (len(args)>1):
filename = args[0].strip()
filelist.append(filename)
fh.close()
### Set variables
boxsize = apFile.getBoxSize(filelist[0])
partperiter = int(1e9/(boxsize[0]**2)/16.)
if partperiter > 4096:
partperiter = 4096
apDisplay.printMsg("Using %d particle per iteration"%(partperiter))
numpart = len(filelist)
if numpart < partperiter:
partperiter = numpart
### Process images
imgnum = 0
stacklist = []
stackroot = stackfile[:-4]
### get memory in kB
startmem = mem.active()
while imgnum < len(filelist):
filename = filelist[imgnum]
index = imgnum % partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
#sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index))
if mem.active()-startmem > 2e6:
apDisplay.printWarning("Out of memory")
if index < 1:
#print "img num", imgnum
### deal with large stacks, reset loop
if imgnum > 0:
sys.stderr.write("\n")
stackname = "%s-%d.hed"%(stackroot, imgnum)
apDisplay.printMsg("writing single particles to file "+stackname)
stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
perpart = (time.time()-starttime)/imgnum
apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"%
(imgnum+1, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart),
apDisplay.timeString(perpart*(numpart-imgnum))), "blue")
stackarray = []
### merge particles
if filetype == "mrc":
partimg = mrc.read(filename)
else:
partimg = spider.read(filename)
stackarray.append(partimg)
imgnum += 1
### write remaining particles to file
sys.stderr.write("\n")
stackname = "%s-%d.hed"%(stackroot, imgnum)
apDisplay.printMsg("writing particles to file "+stackname)
stacklist.append(stackname)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### merge stacks
apFile.removeStack(stackfile, warn=False)
apImagicFile.mergeStacks(stacklist, stackfile)
print stackfile
filepart = apFile.numImagesInStack(stackfile)
if filepart != numpart:
apDisplay.printError("number merged particles (%d) not equal number expected particles (%d)"%
(filepart, numpart))
for stackname in stacklist:
apFile.removeStack(stackname, warn=False)
### summarize
apDisplay.printColor("merged %d particles in %s"%(imgnum, apDisplay.timeString(time.time()-starttime)), "cyan")
def alignStack(oldstack, alignedstack, partlist, dataext=".spi"):
"""
write aligned stack -- with python loop
inputs:
oldstack
newstack (empty)
list of particle dictionaries for operations
modifies:
newstack
output:
none
I tried this loop in both spider and python;
python was faster?!? -neil
"""
if not os.path.isfile(oldstack+dataext):
apDisplay.printError("Could not find original stack: "+oldstack+dataext)
boxsize = apFile.getBoxSize(oldstack+dataext)
apDisplay.printMsg("applying alignment parameters to stack")
apFile.removeFile(alignedstack+dataext)
count = 0
t0 = time.time()
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
#create stack in core
numpart = len(partlist)
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d"%(boxsize), #boxsize
str(numpart+1), #num part to create in memory
str(numpart+1), #max particle number
)
for partdict in partlist:
partnum = partdict['num']
#if partdict['num'] in [3,6,7]:
# print partdict['num'], partdict['template'], partdict['mirror'], round(partdict['rot'],3)
### Rotate and Shift operations
count += 1
#rotate/shift
mySpider.toSpiderQuiet(
"RT SQ",
spyder.fileFilter(oldstack)+"@"+("%06d" % (partnum)),
"_1",
str(partdict['rot']), str(partdict['xshift'])+","+str(partdict['yshift']),
)
#mirror, if necessary
if 'mirror' in partdict and partdict['mirror'] is True:
mySpider.toSpiderQuiet(
"MR", "_1",
"_2@"+("%06d" % (partnum)), "Y",
)
else:
mySpider.toSpiderQuiet(
"CP", "_1",
"_2@"+("%06d" % (partnum)),
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP", "_2@",
spyder.fileFilter(alignedstack)+"@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE", "_2",
)
mySpider.close()
apDisplay.printMsg("Completed transforming %d particles in %s"%(count, apDisplay.timeString(time.time()-t0)))
if count < 1:
apDisplay.printError("Failed to transform any particles")
if not os.path.isfile(alignedstack+dataext):
apDisplay.printError("Failed to create stack "+alignedstack+dataext)
return
def insertRunIntoDatabase(self, alignimagicfile, runparams):
apDisplay.printMsg("Inserting MaxLike Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = runparams['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup max like run
maxlikeq = appiondata.ApMaxLikeRunData()
maxlikeq['runname'] = runparams['runname']
maxlikeq['run_seconds'] = runparams['runtime']
#maxlikeq['mask_diam'] = 2.0*runparams['maskrad']
maxlikeq['fast'] = runparams['fast']
maxlikeq['fastmode'] = runparams['fastmode']
maxlikeq['mirror'] = runparams['mirror']
maxlikeq['student'] = bool(runparams['student'])
maxlikeq['init_method'] = "xmipp default"
maxlikeq['job'] = self.getMaxLikeJob(runparams)
### finish alignment run
alignrunq['maxlikerun'] = maxlikeq
alignrunq['hidden'] = False
alignrunq['runname'] = runparams['runname']
alignrunq['description'] = runparams['description']
alignrunq['lp_filt'] = runparams['lowpass']
alignrunq['hp_filt'] = runparams['highpass']
alignrunq['bin'] = runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = alignimagicfile
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = "part"+self.params['timestamp']+"_average.hed"
alignstackq['iteration'] = self.lastiter
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: "+alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(runparams['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(runparams['stackid'])*runparams['bin']
alignstackq['description'] = runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = runparams['numpart']
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def start(self):
### get stack parameteres
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackId'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackId'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackId'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
### copy stack into working directory
if os.path.isfile(self.stack['file']):
apDisplay.printColor("copying stack into running directoroy", "cyan")
if self.stack['file'][-4:] == ".img" or self.stack['file'][-4:] == ".hed":
strippedfile = self.stack['file'][:-4]
else:
strippedfile = self.stack['file']
while os.path.isfile(os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(os.path.join(self.params['rundir'], "start.img"))
emancmd = "proc2d "+strippedfile+".hed "+os.path.join(self.params['rundir'], "start.hed ")+\
"first=0 last="+str(self.params['numpart']-1)
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("stack not found in database")
### get template stack parameters
self.templatestack = {}
self.templatestack['data'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId'])
self.templatestack['apix'] = self.templatestack['data']['apix']
self.templatestack['boxsize'] = self.templatestack['data']['boxsize']
self.templatestack['file'] = os.path.join(self.templatestack['data']['path']['path'], self.templatestack['data']['templatename'])
self.templatestack['numimages'] = self.templatestack['data']['numimages']
### copy templates into working directory
if os.path.isfile(self.templatestack['file']):
apDisplay.printColor("copying templates into running directoroy", "cyan")
ts = os.path.join(self.params['rundir'], "references.img")
while os.path.isfile(ts):
apFile.removeStack(ts)
if self.templatestack['file'][-4:] == ".img" or self.templatestack['file'][-4:] == ".hed":
strippedfile = self.templatestack['file'][:-4]
else:
strippedfile = self.templatestack['file']
emancmd = "proc2d "+strippedfile+".img "+ts
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("template stack not found in database")
### set new pixelsize
if self.params['bin'] is not None and self.params['bin'] != 0:
self.params['apix'] = float(self.stack['apix']) * int(self.params['bin'])
else:
self.params['apix'] = self.stack['apix']
### scale, low-pass, and high-pass filter stack ... do this with imagic, because it determines the appropriate boxsizes
scalingbatchfile = self.createImagicBatchFileScaling()
preptime = time.time()
proc = subprocess.Popen("chmod 775 "+str(scalingbatchfile), shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(scalingbatchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(os.path.join(self.params['rundir'], "prepareStack.log"))
apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-preptime), "cyan")
### set new boxsize, done only after scaling is complete
if self.params['bin'] is not None:
self.params['boxsize'] = apFile.getBoxSize(os.path.join(self.params['rundir'], "start.hed"))[0]
else:
self.params['boxsize'] = self.stack['boxsize']
### make sure template stack boxsize matches that of the input stack
if self.params['apix'] != self.templatestack['apix'] or self.params['boxsize'] != self.templatestack['boxsize']:
self.scaleTemplates()
starttime=time.time()
print self.params
print "... stack pixel size: "+str(self.params['apix'])
print "... stack box size: "+str(self.params['boxsize'])
apDisplay.printColor("Running IMAGIC .batch file: See multiReferenceAlignment.log file for details", "cyan")
### create IMAGIC batch file
batchfile = self.createImagicBatchFileMRA()
### execute IMAGIC batch file
aligntime0 = time.time()
proc = subprocess.Popen("chmod 775 "+str(batchfile), shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(batchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(os.path.join(self.params['rundir'], "multiReferenceAlignment.log"))
apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-aligntime0), "cyan")
### get particle parameters (shift, rotate, refnum, mirror, ccc)
partparams = self.getParticleParams()
### average stack
alignstack = os.path.join(self.params['rundir'], "alignstack.hed")
apStack.averageStack(alignstack)
### normalize particles (otherwise found problems in viewing with stackviewer)
emancmd = "proc2d "+alignstack+" "+alignstack+".norm.hed norm"
while os.path.isfile(alignstack+".norm.img"):
apFile.removeStack(alignstack+".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(alignstack+".norm.hed", alignstack)
os.rename(alignstack+".norm.img", alignstack[:-4]+".img")
### normalize references
emancmd = "proc2d "+ts+" "+ts+".norm.hed norm"
while os.path.isfile(ts+".norm.img"):
apFile.removeStack(ts+".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(ts+".norm.hed", ts)
os.rename(ts+".norm.img", ts[:-4]+".img")
### remove copied stack
while os.path.isfile(os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(os.path.join(self.params['rundir'], "start.img"))
### insert run into database
self.insertAlignmentRun(insert=True)
self.insertParticlesIntoDatabase(partparams, insert=True)
def insertRunIntoDatabase(self, alignedPartStack, alignedClassStack,
runparams):
apDisplay.printMsg("Inserting ISAC Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = runparams['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + runparams['runname'] +
"' and path already exist in database")
### setup ISAC like run
isacq = appiondata.ApSparxISACRunData()
isacq['runname'] = runparams['runname']
isacq['job'] = self.getISACJobData(runparams)
### finish alignment run
alignrunq['isacrun'] = isacq
alignrunq['hidden'] = False
alignrunq['runname'] = runparams['runname']
alignrunq['description'] = runparams['description']
alignrunq['lp_filt'] = runparams['lowpass']
alignrunq['hp_filt'] = runparams['highpass']
alignrunq['bin'] = runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = alignedPartStack
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = alignedClassStack
alignstackq['iteration'] = self.lastiter
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: " +
alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: " +
refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(runparams['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid']) * self.params['bin']
alignstackq['description'] = runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.numPart
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apParam.runCmd(proccmd, "EMAN", verbose=True)
if self.params['numpart'] != apFile.numImagesInStack(
self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -codes " + str(self.params['numrefs']) + " -iter " +
str(self.params['maxiter']) + " -o " + os.path.join(
self.params['rundir'], "part" + self.params['timestamp']))
if self.params['fast']:
xmippopts += " -fast "
if self.params['correlation']:
xmippopts += " -useCorrelation "
if self.params['classical']:
xmippopts += " -classicalMultiref "
if self.params['align']:
xmippopts += " -alignImages "
### use multi-processor command
apDisplay.printColor(
"Using " + str(self.params['nproc']) + " processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True)
mpiruncmd = self.mpirun + " -np " + str(
self.params['nproc']) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd,
package="Xmipp",
verbose=True,
showcmd=True,
logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(self.params['runtime']))
### minor post-processing
self.createReferenceStack()
self.parseOutput()
self.clearIntermediateFiles()
# self.readyUploadFlag()
apParam.dumpParameters(
self.params, "cl2d-" + self.params['timestamp'] + "-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-" +
self.params['timestamp'] +
"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(
self.runparams['stackid']) * self.runparams['bin']
self.Nlevels = len(
glob.glob("part" + self.params['timestamp'] + "_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part" + self.params[
'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels - 1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part" + self.params['timestamp'] +
"_level_%02d_.hed" % level, classnum + 1,
partdict[classnum], len(partdict))
def filterAndChimera(density,
res=30,
apix=None,
box=None,
chimtype='snapshot',
contour=None,
zoom=1.0,
sym='c1',
color=None,
silhouette=True,
mass=None):
"""
filter volume and then create a few snapshots for viewing on the web
"""
if isValidVolume(density) is False:
apDisplay.printError("Volume file is not valid")
if box is None:
boxdims = apFile.getBoxSize(density)
box = boxdims[0]
### if eotest failed, filter to 30
if not res or str(res) == 'nan':
res = 30
### low pass filter the volume to 60% of reported res
tmpf = density + '.tmp.mrc'
filtres = 0.6 * res
shrinkby = 1
if box is not None and box > 250:
shrinkby = int(math.ceil(box / 160.0))
if box % (2 * shrinkby) == 0:
### box is divisible by shrink by
lpcmd = (
'proc3d %s %s apix=%.3f tlp=%.2f shrink=%d origin=0,0,0 norm=0,1'
% (density, tmpf, apix, filtres, shrinkby))
else:
### box not divisible by shrink by, need a clip
clip = math.floor(box / shrinkby / 2.0) * 2 * shrinkby
lpcmd = (
'proc3d %s %s apix=%.3f tlp=%.2f shrink=%d origin=0,0,0 norm=0,1 clip=%d,%d,%d'
% (density, tmpf, apix, filtres, shrinkby, clip, clip, clip))
else:
lpcmd = ('proc3d %s %s apix=%.3f tlp=%.2f origin=0,0,0 norm=0,1' %
(density, tmpf, apix, filtres))
apDisplay.printMsg("Low pass filtering model for images")
proc = subprocess.Popen(lpcmd, shell=True)
proc.wait()
### flatten solvent
vol = mrc.read(tmpf)
numpy.where(vol < 0, 0.0, vol)
mrc.write(vol, tmpf)
del vol
### contour volume to mass
if mass is not None:
setVolumeMass(tmpf, apix * shrinkby, mass)
contour = 1.0
### set pixelsize and origin
recmd = "proc3d %s %s apix=%.3f origin=0,0,0" % (tmpf, tmpf, apix)
proc = subprocess.Popen(recmd, shell=True)
proc.wait()
### render images
renderSlice(density, box=box, tmpfile=tmpf, sym=sym)
if chimtype != 'snapshot':
renderAnimation(tmpf,
contour,
zoom,
sym,
color,
silhouette,
name=density)
elif chimtype != 'animate':
renderSnapshots(tmpf,
contour,
zoom,
sym,
color,
silhouette,
name=density)
apFile.removeFile(tmpf)
