def getBeamTiltFromStackParticleNumber(self):
stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum+1, msg=True)
pdata = stackparticledata['particle']
### step 1: get axial beam tilt on to image coordinate direction
imageshift = pdata['image']['scope']['image shift']
zerobeamtilt = {'x':0.0e-3, 'y':0.0e-3}
axialbeamtilt = self.transformImageShiftToBeamTilt(imageshift, self.tem, self.cam, self.ht, zerobeamtilt, self.mag)
beamtilt_rotationtransform = self.retrieveBeamTiltMatrix(self.tem, self.cam, self.ht, self.mag)
axialbtvector = numpy.array((axialbeamtilt['y'],axialbeamtilt['x']))
axialbtvector_on_image = numpy.dot(beamtilt_rotationtransform, axialbtvector)
apDisplay.printMsg( "Axial Beam Tilt on Image Coordinate ( %5.2f, %5.2f)" % (axialbtvector_on_image[1]*1e3,axialbtvector_on_image[0]*1e3))
### step 2: get off-axis beam tilt
pcoord = {'x':pdata['xcoord']*self.imgpix,'y':pdata['ycoord']*self.imgpix}
apix = apStack.getStackPixelSizeFromStackId(self.stackid)
camdata = pdata['image']['camera']
cambin = camdata['binning']
camdim = camdata['dimension']
camoff = camdata['offset']
camcenter = {'x':self.camsize['x']/2,'y':self.camsize['y']/2}
apDisplay.printMsg( 'Particle coord (%5d,%5d)' % (pdata['xcoord'],pdata['ycoord']))
mcoord = {'y':pdata['ycoord']*cambin['y']+camoff['y']-camcenter['y'],
'x':pdata['xcoord']*cambin['x']+camoff['x']-camcenter['x']}
apDisplay.printMsg( 'Particle from center in camera pixel (%5d,%5d)' % (mcoord['x'],mcoord['y']))
coord = {'x':mcoord['x']*self.imgpix*cambin['x'],
'y':mcoord['y']*self.imgpix*cambin['y']}
apDisplay.printMsg( 'Particle from center (A) (%5.2f,%5.2f)' % (coord['x']*1e10,coord['y']*1e10))
offaxisbeamtilt = self.getCombinedOffAxisBeamTilt(self.c2diameter,self.beamdiameter,coord)
#offaxisbeamtilt = {'x':0.0,'y':0.0}
### step 3: combine beam tilts
totalbeamtilt = {'x':axialbtvector_on_image[1]+offaxisbeamtilt['x'],
'y':axialbtvector_on_image[0]-offaxisbeamtilt['y'] }
apDisplay.printMsg( "Total Beam Tilt on Image Coordinate (mrad) ( %5.2f, %5.2f)" % (totalbeamtilt['x']*1e3,totalbeamtilt['y']*1e3))
return totalbeamtilt
def setupParticles(self):
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
apix = self.stack['apix']*self.params['bin']
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1:
clipsize = boxsize*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
if self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
proccmd += " last="+str(self.params['numpart']-1)
apFile.removeStack(self.params['localstack'], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
return (boxsize, apix)
def setupParticles(self):
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2
apix = self.stack['apix']*self.params['bin']
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1:
clipsize = boxsize*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
if self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
proccmd += " last="+str(self.params['numpart']-1)
apFile.removeStack(self.params['localstack'], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
return (boxsize, apix)
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.checkNoRefRun()
### convert stack to spider
spiderstack = self.createSpiderFile()
### create initialization template
if self.params['initmethod'] == 'allaverage':
templatefile = self.averageTemplate()
elif self.params['initmethod'] == 'selectrand':
templatefile = self.selectRandomParticles()
elif self.params['initmethod'] == 'randpart':
templatefile = self.pickRandomParticle()
elif self.params['initmethod'] == 'template':
templatefile = self.getTemplate()
else:
apDisplay.printError("unknown initialization method defined: "
+str(self.params['initmethod'])+" not in "+str(self.initmethods))
apDisplay.printColor("Running spider this can take awhile","cyan")
### run the alignment
aligntime = time.time()
pixrad = int(round(self.params['partrad']/self.stack['apix']/self.params['bin']))
alignedstack, self.partlist = alignment.refFreeAlignParticles(
spiderstack, templatefile,
self.params['numpart'], pixrad,
self.params['firstring'], self.params['lastring'],
rundir = ".")
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=False)
### convert stack to imagic
imagicstack = self.convertSpiderStack(alignedstack)
apFile.removeFile(alignedstack)
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = aligntime
self.insertNoRefRun(imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def setupParticles(self):
self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed")
self.stack = {}
self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"])
self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"])
self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"])
self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"])
boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2
apix = self.stack["apix"] * self.params["bin"]
proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"])
if self.params["bin"] > 1:
clipsize = boxsize * self.params["bin"]
proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize)
if self.params["highpass"] > 1:
proccmd += " hp=" + str(self.params["highpass"])
if self.params["lowpass"] > 1:
proccmd += " lp=" + str(self.params["lowpass"])
proccmd += " last=" + str(self.params["numpart"] - 1)
apFile.removeStack(self.params["localstack"], warn=False)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"])
return (boxsize, apix)
def getBeamTiltFromStackParticleNumber(self):
stackparticledata = apStack.getOneParticleFromStackId(self.stackid,
self.imgnum + 1,
msg=True)
pdata = stackparticledata['particle']
### step 1: get axial beam tilt on to image coordinate direction
imageshift = pdata['image']['scope']['image shift']
zerobeamtilt = {'x': 0.0e-3, 'y': 0.0e-3}
axialbeamtilt = self.transformImageShiftToBeamTilt(
imageshift, self.tem, self.cam, self.ht, zerobeamtilt, self.mag)
beamtilt_rotationtransform = self.retrieveBeamTiltMatrix(
self.tem, self.cam, self.ht, self.mag)
axialbtvector = numpy.array((axialbeamtilt['y'], axialbeamtilt['x']))
axialbtvector_on_image = numpy.dot(beamtilt_rotationtransform,
axialbtvector)
apDisplay.printMsg(
"Axial Beam Tilt on Image Coordinate ( %5.2f, %5.2f)" %
(axialbtvector_on_image[1] * 1e3, axialbtvector_on_image[0] * 1e3))
### step 2: get off-axis beam tilt
pcoord = {
'x': pdata['xcoord'] * self.imgpix,
'y': pdata['ycoord'] * self.imgpix
}
apix = apStack.getStackPixelSizeFromStackId(self.stackid)
camdata = pdata['image']['camera']
cambin = camdata['binning']
camdim = camdata['dimension']
camoff = camdata['offset']
camcenter = {'x': self.camsize['x'] / 2, 'y': self.camsize['y'] / 2}
apDisplay.printMsg('Particle coord (%5d,%5d)' %
(pdata['xcoord'], pdata['ycoord']))
mcoord = {
'y': pdata['ycoord'] * cambin['y'] + camoff['y'] - camcenter['y'],
'x': pdata['xcoord'] * cambin['x'] + camoff['x'] - camcenter['x']
}
apDisplay.printMsg('Particle from center in camera pixel (%5d,%5d)' %
(mcoord['x'], mcoord['y']))
coord = {
'x': mcoord['x'] * self.imgpix * cambin['x'],
'y': mcoord['y'] * self.imgpix * cambin['y']
}
apDisplay.printMsg('Particle from center (A) (%5.2f,%5.2f)' %
(coord['x'] * 1e10, coord['y'] * 1e10))
offaxisbeamtilt = self.getCombinedOffAxisBeamTilt(
self.c2diameter, self.beamdiameter, coord)
#offaxisbeamtilt = {'x':0.0,'y':0.0}
### step 3: combine beam tilts
totalbeamtilt = {
'x': axialbtvector_on_image[1] + offaxisbeamtilt['x'],
'y': axialbtvector_on_image[0] - offaxisbeamtilt['y']
}
apDisplay.printMsg(
"Total Beam Tilt on Image Coordinate (mrad) ( %5.2f, %5.2f)" %
(totalbeamtilt['x'] * 1e3, totalbeamtilt['y'] * 1e3))
return totalbeamtilt
def getInstrumentParams(self):
self.stackpix = apStack.getStackPixelSizeFromStackId(self.stackid, True) * 1e-10
stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum+1, msg=True)
pdata = stackparticledata['particle']
self.imgpix = apDatabase.getPixelSize(pdata['image'])*1e-10
scopedata = pdata['image']['scope']
self.tem = scopedata['tem']
self.cam = pdata['image']['camera']['ccdcamera']
self.ht = scopedata['high tension']
self.Cs = 2e-3
self.mag = scopedata['magnification']
self.camsize = {'x':4096,'y':4096}
self.beamdiameter = 1e-6
self.c2diameter = 1.0e-4
self.wavelength = fftfun.getElectronWavelength(self.ht)
def getInstrumentParams(self):
self.stackpix = apStack.getStackPixelSizeFromStackId(
self.stackid, True) * 1e-10
stackparticledata = apStack.getOneParticleFromStackId(self.stackid,
self.imgnum + 1,
msg=True)
pdata = stackparticledata['particle']
self.imgpix = apDatabase.getPixelSize(pdata['image']) * 1e-10
scopedata = pdata['image']['scope']
self.tem = scopedata['tem']
self.cam = pdata['image']['camera']['ccdcamera']
self.ht = scopedata['high tension']
self.Cs = 2e-3
self.mag = scopedata['magnification']
self.camsize = {'x': 4096, 'y': 4096}
self.beamdiameter = 1e-6
self.c2diameter = 1.0e-4
self.wavelength = fftfun.getElectronWavelength(self.ht)
def createFastFreeHandInputLineTemplate(self):
self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.apix = 6.02
one_particledata = apStack.getOneParticleFromStackId(self.params['stackid'], particlenumber=1, msg=True)
scopedata = one_particledata['particle']['image']['scope']
ht_kv = scopedata['high tension'] / 1000.0
cs_mm = scopedata['tem']['cs'] * 1000
angSearch = self.params['angSearch']
constant_inputlines = [
'%.3f,%.3f,%2f,%.f' %(self.apix,self.params['snr'],cs_mm,ht_kv)
,'1,0'
,'../%s' %(self.stackfile)
,'../%s' %(self.modelfile)
,'../%s' %(self.frealign_paramfile)
,'out.mrc'
,'-%d,%d,-%d,%d' %(angSearch,angSearch,angSearch,angSearch)
,'%.1f,%.2f,%.2f' %(self.params['minres'],self.params['maxres'],self.params['radius'])
,'%s' %(self.params['scoringtype'].upper())
]
return constant_inputlines
def createFastFreeHandInputLineTemplate(self):
self.boxsize = apStack.getStackBoxsize(self.params["stackid"], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
self.apix = 6.02
one_particledata = apStack.getOneParticleFromStackId(self.params["stackid"], particlenumber=1, msg=True)
scopedata = one_particledata["particle"]["image"]["scope"]
ht_kv = scopedata["high tension"] / 1000.0
cs_mm = scopedata["tem"]["cs"] * 1000
angSearch = self.params["angSearch"]
constant_inputlines = [
"%.3f,%.3f,%2f,%.f" % (self.apix, self.params["snr"], cs_mm, ht_kv),
"1,0",
"../%s" % (self.stackfile),
"../%s" % (self.modelfile),
"../%s" % (self.frealign_paramfile),
"out.mrc",
"-%d,%d,-%d,%d" % (angSearch, angSearch, angSearch, angSearch),
"%.1f,%.2f,%.2f" % (self.params["minres"], self.params["maxres"], self.params["radius"]),
"%s" % (self.params["scoringtype"].upper()),
]
return constant_inputlines
def checkConflicts(self):
### first get all stack data
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
### modify boxsize and pixelsize according to binning
self.boxsize = self.stack['boxsize']
self.clipsize = int(math.floor(self.boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
self.apix = self.stack['apix']
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize)*self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
self.boxsize = clipsize
self.apix = self.apix * self.params['bin']
### basic error checking
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['ncls'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 1000000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(stackfile)))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > 5000:
apDisplay.printWarning("initial model calculation may not work with less than 5000 particles")
self.mpirun = self.checkMPI()
# if self.mpirun is None:
# apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
### SIMPLE defaults and error checking
if self.params['ring2'] is None:
self.params['ring2'] = (self.boxsize/2) - 2
if self.params['ncls'] > 2000:
apDisplay.printError("number of classes should be less than 2000 for subsequent ORIGAMI run to work")
if self.params['ncls'] > self.params['numpart']:
self.params['ncls'] = self.params['numpart'] / self.params['minp']
if self.params['mask'] is None:
self.params['mask'] = (self.boxsize/2) - 2
if self.params['mw'] is None:
apDisplay.printError("please specify the molecular weight (in kDa)")
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.checkNoRefRun()
### convert stack to spider
spiderstack = self.createSpiderFile()
### create initialization template
if self.params['initmethod'] == 'allaverage':
templatefile = self.averageTemplate()
elif self.params['initmethod'] == 'selectrand':
templatefile = self.selectRandomParticles()
elif self.params['initmethod'] == 'randpart':
templatefile = self.pickRandomParticle()
elif self.params['initmethod'] == 'template':
templatefile = self.getTemplate()
else:
apDisplay.printError("unknown initialization method defined: " +
str(self.params['initmethod']) + " not in " +
str(self.initmethods))
apDisplay.printColor("Running spider this can take awhile", "cyan")
### run the alignment
aligntime = time.time()
pixrad = int(
round(self.params['partrad'] / self.stack['apix'] /
self.params['bin']))
alignedstack, self.partlist = alignment.refFreeAlignParticles(
spiderstack,
templatefile,
self.params['numpart'],
pixrad,
self.params['firstring'],
self.params['lastring'],
rundir=".")
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: " + spiderstack)
apFile.removeFile(spiderstack, warn=False)
### convert stack to imagic
imagicstack = self.convertSpiderStack(alignedstack)
apFile.removeFile(alignedstack)
inserttime = time.time()
if self.params['commit'] is True:
self.runtime = aligntime
self.insertNoRefRun(imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: " +
apDisplay.timeString(inserttime))
def getSessionDataFromReconId(reconid):
stackid = apStack.getStackIdFromRecon(reconid)
partdata = apStack.getOneParticleFromStackId(stackid, msg=False)
sessiondata = partdata['particle']['selectionrun']['session']
return sessiondata
def start(self):
self.addToLog('.... Setting up new ISAC job ....')
self.addToLog('.... Making command for stack pre-processing ....')
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### send file to remotehost
tasks = {}
sfhed = self.stack['file'][:-4] + ".hed"
sfimg = self.stack['file'][:-4] + ".img"
tasks = self.addToTasks(
tasks,
"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s"
% (sfhed, self.params['localhost'], self.params['remoterundir'],
"start1.hed"))
tasks = self.addToTasks(
tasks,
"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s"
% (sfimg, self.params['localhost'], self.params['remoterundir'],
"start1.img"))
### write Sparx jobfile: process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
# apParam.runCmd(proccmd, "EMAN", verbose=True)
self.addSimpleCommand('cd %s' % self.params['rundir'])
self.addSimpleCommand(proccmd)
sparxcmd = "sxcpy.py %s %s_1.hdf" % (self.params['localstack'],
self.params['localstack'][:-4])
# apParam.runCmd(sparxcmd, "SPARX", verbose=True)
self.addSimpleCommand(sparxcmd)
self.addSimpleCommand("")
### write Sparx jobfile: run ISAC
for i in range(self.params['generations']):
sparxopts = " %s_%d.hdf" % (os.path.join(
self.params['localstack'][:-4]), (i + 1))
if self.params['ir'] is not None:
sparxopts += " --ir %d" % int(
float(
apRecon.getComponentFromVector(
self.params['ir'], i, splitter=":")))
if self.params['ou'] is not None:
sparxopts += " --ou %d" % int(
float(
apRecon.getComponentFromVector(
self.params['ou'], i, splitter=":")))
if self.params['rs'] is not None:
sparxopts += " --rs %d" % int(
float(
apRecon.getComponentFromVector(
self.params['rs'], i, splitter=":")))
if self.params['ts'] is not None:
sparxopts += " --ts %.1f" % int(
float(
apRecon.getComponentFromVector(
self.params['ts'], i, splitter=":")))
if self.params['xr'] is not None:
sparxopts += " --xr %.1f" % int(
float(
apRecon.getComponentFromVector(
self.params['xr'], i, splitter=":")))
if self.params['yr'] is not None:
sparxopts += " --yr %.1f" % int(
float(
apRecon.getComponentFromVector(
self.params['yr'], i, splitter=":")))
if self.params['maxit'] is not None:
sparxopts += " --maxit %d" % int(
float(
apRecon.getComponentFromVector(
self.params['maxit'], i, splitter=":")))
if self.params['FL'] is not None:
sparxopts += " --FL %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['FL'], i, splitter=":")))
if self.params['FH'] is not None:
sparxopts += " --FH %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['FH'], i, splitter=":")))
if self.params['FF'] is not None:
sparxopts += " --FF %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['FF'], i, splitter=":")))
if self.params['init_iter'] is not None:
sparxopts += " --init_iter %d" % int(
float(
apRecon.getComponentFromVector(
self.params['init_iter'], i, splitter=":")))
if self.params['main_iter'] is not None:
sparxopts += " --main_iter %d" % int(
float(
apRecon.getComponentFromVector(
self.params['main_iter'], i, splitter=":")))
if self.params['iter_reali'] is not None:
sparxopts += " --iter_reali %d" % int(
float(
apRecon.getComponentFromVector(
self.params['iter_reali'], i, splitter=":")))
if self.params['match_first'] is not None:
sparxopts += " --match_first %d" % int(
float(
apRecon.getComponentFromVector(
self.params['match_first'], i, splitter=":")))
if self.params['max_round'] is not None:
sparxopts += " --max_round %d" % int(
float(
apRecon.getComponentFromVector(
self.params['max_round'], i, splitter=":")))
if self.params['match_second'] is not None:
sparxopts += " --match_second %d" % int(
float(
apRecon.getComponentFromVector(
self.params['match_second'], i, splitter=":")))
if self.params['stab_ali'] is not None:
sparxopts += " --stab_ali %d" % int(
float(
apRecon.getComponentFromVector(
self.params['stab_ali'], i, splitter=":")))
if self.params['thld_err'] is not None:
sparxopts += " --thld_err %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['thld_err'], i, splitter=":")))
if self.params['indep_run'] is not None:
sparxopts += " --indep_run %d" % int(
float(
apRecon.getComponentFromVector(
self.params['indep_run'], i, splitter=":")))
if self.params['thld_grp'] is not None:
sparxopts += " --thld_grp %d" % int(
float(
apRecon.getComponentFromVector(
self.params['thld_grp'], i, splitter=":")))
if self.params['img_per_grp'] is not None:
sparxopts += " --img_per_grp %d" % int(
apRecon.getComponentFromVector(self.params['img_per_grp'],
i))
sparxopts += " --generation %d" % (i + 1)
sparxexe = apParam.getExecPath("sxisac.py", die=True)
mpiruncmd = self.mpirun + " -np " + str(
self.params['nproc']) + " " + sparxexe + " " + sparxopts
bn = os.path.basename(self.params['localstack'])[:-4]
e2cmd = "e2proc2d.py %s_%d.hdf %s_%d.hdf --list=\"generation_%d_unaccounted.txt\"" % \
(bn, i+1, bn, i+2, i+1)
self.addSimpleCommand(mpiruncmd)
self.addSimpleCommand(e2cmd)
# print self.tasks
# print self.command_list
# self.writeCommandListToFile()
apParam.dumpParameters(
self.params, "isac-" + self.params['timestamp'] + "-params.pickle")
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local temp file
proccmd = "proc2d " + self.stack['file'] + " temp.hed apix=" + str(
self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### process stack to final file
self.params['localstack'] = os.path.join(self.params['rundir'],
self.timestamp + ".hed")
proccmd = "proc2d temp.hed " + self.params[
'localstack'] + " apix=" + str(
self.stack['apix'] * self.params['bin'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
apFile.removeStack("temp.hed")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### convert stack into single spider files
templateselfile = self.initializeTemplates()
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -iter " + str(self.params['maxiter']) + " -ref " +
templateselfile + " -o " +
os.path.join(self.params['rundir'], "part" + self.timestamp) +
" -psi_step " + str(self.params['psistep']))
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-18 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using " + str(nproc - 1) + " processors!",
"green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun + " -np " + str(
nproc - 1) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe + " " + xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
### test insert to make sure data is not overwritten
self.params['runtime'] = 0
#self.checkDuplicateRefBasedRun()
### set box size
self.boxsize = int(math.floor(self.stack['boxsize']/self.params['bin']/2.0))*2
### convert stack to spider
spiderstack = self.createSpiderFile()
### create template stack
templatestack = self.createTemplateStack()
### run the alignment
aligntime = time.time()
usestack = spiderstack
oldpartlist = None
for i in range(self.params['numiter']):
iternum = i+1
apDisplay.printColor("\n\nITERATION "+str(iternum), "green")
alignedstack, partlist = self.runAlignmentGPU(
usestack, templatestack, spiderstack,
self.params['xysearch'], self.params['xystep'],
self.params['firstring'], self.params['lastring'],
iternum=iternum, oldpartlist=oldpartlist)
oldpartlist = partlist
usestack = alignedstack
templatestack = self.updateTemplateStack(alignedstack, partlist, iternum)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=True)
### convert aligned stack to imagic
finalspistack = "aligned.spi"
shutil.move(alignedstack, finalspistack)
imagicstack = "aligned.hed"
apFile.removeStack(imagicstack)
emancmd = "proc2d "+finalspistack+" "+imagicstack
apEMAN.executeEmanCmd(emancmd, verbose=True)
### average stack
apStack.averageStack(imagicstack)
### calculate resolution for each reference
apix = self.stack['apix']*self.params['bin']
self.calcResolution(partlist, imagicstack, apix)
if self.params['commit'] is True:
apDisplay.printMsg("committing results to DB")
self.params['runtime'] = aligntime
self.insertRefBasedRun(partlist, imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
### remove temporary files
apFile.removeFilePattern("alignments/alignedstack*.spi")
apFile.removeFile(finalspistack)
def convertToRefineStack(self):
'''
The stack is remaked without ctf correction and without invertion (ccd)
'''
newstackroot = os.path.join(self.params['rundir'],
os.path.basename(self.stack['file'])[:-4])
if self.params['paramonly'] is True:
print 'newstackroot', newstackroot
self.setFrealignStack(newstackroot)
return
# copy existing refined stack if possible
existing_refine_stack = apStack.getExistingRefineStack(
self.stack['data'],
'frealign',
False,
self.params['last'],
self.params['bin'],
lowpass=self.params['lowpass'],
highpass=self.params['highpass'])
try:
if existing_refine_stack:
if existing_refine_stack != os.path.join(
self.params['rundir'], newstackroot + '.mrc'):
shutil.copyfile(existing_refine_stack,
newstackroot + '.mrc')
new_partorderfile = os.path.join(self.params['rundir'],
'stackpartorder.list')
existing_partorderfile = os.path.join(
os.path.dirname(existing_refine_stack),
'stackpartorder.list')
# particle order list also need to be copied
if not os.path.isfile(new_partorderfile) and os.path.isfile(
existing_partorderfile):
shutil.copyfile(existing_partorderfile, new_partorderfile)
self.setFrealignStack(newstackroot)
return
except:
# make one from scratch if there is error
apDisplay.printMsg(
'Error copying existing refine stack, will regenerate')
if self.no_ctf_correction:
self.ImagicStackToFrealignMrcStack(self.stack['file'])
self.setFrealignStack(newstackroot)
return
# stack need to be remade without ctf correction
apDisplay.printWarning(
'Frealign needs a stack without ctf correction. A new stack is being made....'
)
stackdata = self.stack['data']
madestackdata = self.getOriginalStackMadeByMakeStack()
stackid = stackdata.dbid
stackruns = apStack.getStackRunsFromStack(madestackdata)
# To Do: need to handle combined stack
stackrun = stackruns[0]
stackpathname = os.path.basename(stackdata['path']['path'])
totalpart = apStack.getNumberStackParticlesFromId(stackid)
if not self.params['last']:
numpart = totalpart
else:
numpart = min(self.params['last'], totalpart)
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir, 'temp.hed')
# use first particle image to get presetname
oneparticle = apStack.getOneParticleFromStackId(stackid,
particlenumber=1)
preset = oneparticle['particle']['image']['preset']
if preset:
presetname = preset['name']
else:
presetname = 'manual'
# use first stack run to get parameters
paramdata = stackrun['stackParams']
# binning is combination of the original binning of the stack and the preparation binnning
bin = paramdata['bin'] * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * paramdata['bin']
lowpasstext = self.setArgText(
'lowpass', (self.params['lowpass'], paramdata['lowpass']), False)
highpasstext = self.setArgText(
'highpass', (self.params['highpass'], paramdata['highpass']), True)
partlimittext = self.setArgText('partlimit', (numpart, ), False)
#normalization
normtext = ''
if 'xmipp-norm' in paramdata.keys():
# myami-3.1 or before
normtext = self.setArgText('xmipp-normalize',
(paramdata['xmipp-norm'], ), True)
normtext = '--normalized ' + normtext
elif 'normalizemethod' in paramdata.keys():
# myami-3.2 or newer
normtext = '--normalize-method=%s' % (
paramdata['normalizemethod'], )
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
# The assumption is that the image is from ice grid and digital camera (black particles on white background
if 'reverse' in self.stackrunlogparams.keys():
reversetext = '--reverse'
else:
reversetext = ''
if 'defocpair' in self.stackrunlogparams.keys():
defoctext = '--defocpair'
else:
defoctext = ''
cmd = '''
makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s --no-invert %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2
''' % (os.path.basename(newstackimagicfile), stackid, lowpasstext,
highpasstext, partlimittext, reversetext, defoctext, normtext,
unbinnedboxsize, bin, stackpathname, stackid, projectid, presetname,
newstackrunname, newstackrundir, sessionname, sessionid)
logfilepath = os.path.join(newstackrundir, 'frealignstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd, logfilepath)
if returncode > 0:
apDisplay.printError('Error in Frealign specific stack making')
self.ImagicStackToFrealignMrcStack(newstackimagicfile)
os.rename(newstackimagicfile[:-4] + '.mrc', newstackroot + '.mrc')
self.setFrealignStack(newstackroot)
# use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
#clean up
rmfiles = glob.glob("*.box")
for rmfile in rmfiles:
apFile.removeFile(rmfile)
def getSessionDataFromReconId(reconid):
stackid = apStack.getStackIdFromRecon(reconid)
partdata = apStack.getOneParticleFromStackId(stackid, msg=False)
sessiondata = partdata["particle"]["selectionrun"]["session"]
return sessiondata
def start(self):
self.addToLog('.... Setting up new ISAC job ....')
self.addToLog('.... Making command for stack pre-processing ....')
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
### send file to remotehost
tasks = {}
sfhed = self.stack['file'][:-4]+".hed"
sfimg = self.stack['file'][:-4]+".img"
tasks = self.addToTasks(tasks,"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s" % (sfhed,self.params['localhost'],self.params['remoterundir'],"start1.hed"))
tasks = self.addToTasks(tasks,"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s" % (sfimg,self.params['localhost'],self.params['remoterundir'],"start1.img"))
### write Sparx jobfile: process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize)*self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
proccmd += " last="+str(self.params['numpart']-1)
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
# apParam.runCmd(proccmd, "EMAN", verbose=True)
self.addSimpleCommand('cd %s' % self.params['rundir'])
self.addSimpleCommand(proccmd)
sparxcmd = "sxcpy.py %s %s_1.hdf" % (self.params['localstack'], self.params['localstack'][:-4])
# apParam.runCmd(sparxcmd, "SPARX", verbose=True)
self.addSimpleCommand(sparxcmd)
self.addSimpleCommand("")
### write Sparx jobfile: run ISAC
for i in range(self.params['generations']):
sparxopts = " %s_%d.hdf" % (os.path.join(self.params['localstack'][:-4]), (i+1))
if self.params['ir'] is not None:
sparxopts += " --ir %d" % int(float(apRecon.getComponentFromVector(self.params['ir'], i, splitter=":")))
if self.params['ou'] is not None:
sparxopts += " --ou %d" % int(float(apRecon.getComponentFromVector(self.params['ou'], i, splitter=":")))
if self.params['rs'] is not None:
sparxopts += " --rs %d" % int(float(apRecon.getComponentFromVector(self.params['rs'], i, splitter=":")))
if self.params['ts'] is not None:
sparxopts += " --ts %.1f" % int(float(apRecon.getComponentFromVector(self.params['ts'], i, splitter=":")))
if self.params['xr'] is not None:
sparxopts += " --xr %.1f" % int(float(apRecon.getComponentFromVector(self.params['xr'], i, splitter=":")))
if self.params['yr'] is not None:
sparxopts += " --yr %.1f" % int(float(apRecon.getComponentFromVector(self.params['yr'], i, splitter=":")))
if self.params['maxit'] is not None:
sparxopts += " --maxit %d" % int(float(apRecon.getComponentFromVector(self.params['maxit'], i, splitter=":")))
if self.params['FL'] is not None:
sparxopts += " --FL %.2f" % int(float(apRecon.getComponentFromVector(self.params['FL'], i, splitter=":")))
if self.params['FH'] is not None:
sparxopts += " --FH %.2f" % int(float(apRecon.getComponentFromVector(self.params['FH'], i, splitter=":")))
if self.params['FF'] is not None:
sparxopts += " --FF %.2f" % int(float(apRecon.getComponentFromVector(self.params['FF'], i, splitter=":")))
if self.params['init_iter'] is not None:
sparxopts += " --init_iter %d" % int(float(apRecon.getComponentFromVector(self.params['init_iter'], i, splitter=":")))
if self.params['main_iter'] is not None:
sparxopts += " --main_iter %d" % int(float(apRecon.getComponentFromVector(self.params['main_iter'], i, splitter=":")))
if self.params['iter_reali'] is not None:
sparxopts += " --iter_reali %d" % int(float(apRecon.getComponentFromVector(self.params['iter_reali'], i, splitter=":")))
if self.params['match_first'] is not None:
sparxopts += " --match_first %d" % int(float(apRecon.getComponentFromVector(self.params['match_first'], i, splitter=":")))
if self.params['max_round'] is not None:
sparxopts += " --max_round %d" % int(float(apRecon.getComponentFromVector(self.params['max_round'], i, splitter=":")))
if self.params['match_second'] is not None:
sparxopts += " --match_second %d" % int(float(apRecon.getComponentFromVector(self.params['match_second'], i, splitter=":")))
if self.params['stab_ali'] is not None:
sparxopts += " --stab_ali %d" % int(float(apRecon.getComponentFromVector(self.params['stab_ali'], i, splitter=":")))
if self.params['thld_err'] is not None:
sparxopts += " --thld_err %.2f" % int(float(apRecon.getComponentFromVector(self.params['thld_err'], i, splitter=":")))
if self.params['indep_run'] is not None:
sparxopts += " --indep_run %d" % int(float(apRecon.getComponentFromVector(self.params['indep_run'], i, splitter=":")))
if self.params['thld_grp'] is not None:
sparxopts += " --thld_grp %d" % int(float(apRecon.getComponentFromVector(self.params['thld_grp'], i, splitter=":")))
if self.params['img_per_grp'] is not None:
sparxopts += " --img_per_grp %d" % int(apRecon.getComponentFromVector(self.params['img_per_grp'], i))
sparxopts += " --generation %d" % (i+1)
sparxexe = apParam.getExecPath("sxisac.py", die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+sparxexe+" "+sparxopts
bn = os.path.basename(self.params['localstack'])[:-4]
e2cmd = "e2proc2d.py %s_%d.hdf %s_%d.hdf --list=\"generation_%d_unaccounted.txt\"" % \
(bn, i+1, bn, i+2, i+1)
self.addSimpleCommand(mpiruncmd)
self.addSimpleCommand(e2cmd)
# print self.tasks
# print self.command_list
# self.writeCommandListToFile()
apParam.dumpParameters(self.params, "isac-"+self.params['timestamp']+"-params.pickle")
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
# symmetry info
if self.params['sym'] == 'Icos':
self.params['symtype'] = 'I'
self.params['symfold'] = None
else:
self.params['symtype'] = self.params['sym'][0]
self.params['symfold'] = int(self.params['sym'][1:])
# eman "d" symmetry is spider "ci"
if self.params['symtype'].upper() == "D":
self.params['symtype'] = "CI"
# create symmetry doc file
sydoc = "sym.spi"
refine.symmetryDoc(self.params['symtype'], self.params['symfold'],
sydoc)
# convert incr to array of increments
ang_inc = self.params['incr'].split(',')
self.params['numiter'] = len(ang_inc)
self.params['increments'] = []
for i in range(0, self.params['numiter']):
self.params['increments'].append(int(ang_inc[i]))
# convert stack to spider stack
spiderstack = os.path.join(self.params['rundir'], 'start.spi')
operations.stackToSpiderStack(self.stack['file'],
spiderstack,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
numpart=self.params['numpart'])
# create filtered stack
spiderstackfilt = spiderstack
if (self.params['lowpass'] + self.params['highpass']) > 0:
spiderstackfilt = os.path.join(self.params['rundir'],
'start_filt.spi')
operations.stackToSpiderStack(self.stack['file'],
spiderstackfilt,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
lp=self.params['lowpass'],
hp=self.params['highpass'],
numpart=self.params['numpart'])
# rescale initial model if necessary
outvol = os.path.join(self.params['rundir'], "vol000.spi")
apModel.rescaleModel(self.params['modelid'],
outvol,
self.stack['boxsize'],
self.stack['apix'],
spider=True)
self.params['itervol'] = outvol
for iter in range(1, self.params['numiter'] + 1):
# create projections for projection matching
apDisplay.printMsg("creating reference projections of volume: %s" %
self.params['itervol'])
projs, numprojs, ang, sel = refine.createProjections(
incr=self.params['increments'][iter - 1],
boxsz=self.stack['boxsize'],
symfold=self.params['symfold'],
invol=self.params['itervol'],
rad=self.params['rad'],
)
# run reference-based alignment
apDisplay.printMsg("running reference-based alignment (AP MQ)")
apmqfile = "apmq%03d.spi" % iter
outang = "angular%03d.spi" % iter
shf = "shifts%03d.spi" % iter
shiftedStack = "parts_shifted.spi"
apFile.removeFile(shf)
refine.spiderAPMQ(
projs=projs,
numprojs=numprojs,
tsearch=self.params['xysearch'],
tstep=self.params['xystep'],
firstRing=self.params['firstring'],
lastRing=self.params['lastring'],
stackfile=spiderstackfilt,
nump=self.params['numpart'],
ang=ang,
apmqfile=apmqfile,
outang=outang,
nproc=self.params['proc'],
)
# use cross-correlation to find the sub-pixel alignment
# of the particles,
# results will be saved in "peakfile.spi"
apFile.removeFile(shiftedStack)
# don't use MPI here - for some reason slower?
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
apmqlist = refine.readDocFile(apmqfile)
avgccrot = 0
apDisplay.printMsg("creating shifted stack")
for p in range(0, self.params['numpart']):
ref = int(float(apmqlist[p][2]))
ccrot = float(apmqlist[p][3])
inplane = float(apmqlist[p][4])
avgccrot += ccrot
# invert the sign - ref projs will be rotated
inplane *= -1
# get corresponding projection
if (ref <= 0):
# mirror projection if necessary
ref *= -1
refimg = spyder.fileFilter(projs) + "@" + str(ref)
refine.mirrorImg(refimg, "_3", inMySpi=mySpi)
img = "_3"
else:
img = spyder.fileFilter(projs) + "@" + str(ref)
refine.rotAndShiftImg(img, "_2", inplane, inMySpi=mySpi)
refine.maskImg("_2",
"_3",
self.params['rad'],
"D",
"E",
center=int((self.stack['boxsize'] / 2) + 1),
inMySpi=mySpi)
# pad ref image & stack image to twice the size
refine.padImg("_3",
"_2",
2 * self.stack['boxsize'],
"N",
1,
1,
0,
inMySpi=mySpi)
stackimg = spyder.fileFilter(spiderstack) + "@" + str(p + 1)
refine.padImg(stackimg,
"_1",
2 * self.stack['boxsize'],
"B",
1,
1,
inMySpi=mySpi)
# calculate cross-correlation
refine.getCC("_1", "_2", "_1", inMySpi=mySpi)
# crop the correllation image to allowable shift amount
shift = int(self.params['allowedShift'] *
self.stack['boxsize'])
dim = 2 * shift + 1
topleftx = self.stack['boxsize'] - shift + 1
refine.windowImg("_1",
"_2",
dim,
topleftx,
topleftx,
inMySpi=mySpi)
# find the sub-pixel location of cc peak
mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17", "_2",
"0")
# create new stack of shifted particles
shpos = spyder.fileFilter(shiftedStack) + "@" + str(p + 1)
mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN")
mySpi.toSpiderQuiet("GP x17", "_2",
str(shift + 1) + "," + str(shift + 1))
refine.copyImg(stackimg, shpos, inMySpi=mySpi)
mySpi.toSpiderQuiet("ELSE")
#mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16")
mySpi.toSpiderQuiet("SH F", stackimg, shpos, "-x15,-x16")
mySpi.toSpiderQuiet("ENDIF")
# save shifts to file
mySpi.toSpiderQuiet("SD " + str(p + 1) + ",x15,x16,x17",
spyder.fileFilter(shf))
mySpi.toSpiderQuiet("SD E", spyder.fileFilter(shf))
mySpi.close()
# create class average images
refine.createClassAverages(
shiftedStack,
projs,
apmqfile,
numprojs,
self.stack['boxsize'],
shifted=True,
)
# rename class averages & variacnes for iteration
cmd = "/bin/mv classes.hed classes.%d.hed;" % iter
cmd += "/bin/mv classes.img classes.%d.img;" % iter
cmd += "/bin/mv variances.hed variances.%d.hed;" % iter
cmd += "/bin/mv variances.img variances.%d.img;" % iter
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
# calculate the stddev for the apmq cc's for throwing out particles
avgccrot /= self.params['numpart']
stdccrot = 0
for p in range(0, self.params['numpart']):
stdccrot += abs(float(apmqlist[p][3]) - avgccrot)
stdccrot /= self.params['numpart']
cccutoff = avgccrot + (stdccrot * self.params['keepsig'])
apDisplay.printMsg("average cc: %f" % avgccrot)
apDisplay.printMsg("setting cutoff to: %f" % cccutoff)
# create new selection file that only has particles with good cc's
selectfile = "select%03d.spi" % iter
apFile.removeFile(selectfile)
mySpi = spyder.SpiderSession(nproc=self.params['proc'],
dataext=".spi",
logo=False,
log=False)
i = 1
for p in range(0, self.params['numpart']):
ccrot = float(apmqlist[p][3])
if ccrot >= cccutoff:
mySpi.toSpiderQuiet("x11=%d" % (p + 1))
mySpi.toSpiderQuiet("SD %d,x11" % i,
spyder.fileFilter(selectfile))
i += 1
mySpi.close()
# calculate the new 3d structure using centered projections
# and the corrected angles from the angular doc file
apDisplay.printMsg("creating 3d volume")
out_rawvol = "vol_raw%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
selectfile,
rad=self.params['rad'],
ang=outang,
out=out_rawvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['bpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'],
)
else:
refine.backProjection(shiftedStack,
selectfile,
ang=outang,
out=out_rawvol,
sym=sydoc,
nproc=self.params['proc'])
# create even & odd select files
apDisplay.printMsg("creating even/odd volumes")
oddfile = "selectodd%03d.spi" % iter
evenfile = "selecteven%03d.spi" % iter
refine.docSplit(selectfile, oddfile, evenfile)
# get the even & odd volumesa
oddvol = "vol1%03d.spi" % iter
evenvol = "vol2%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
oddfile,
rad=self.params['rad'],
ang=outang,
out=oddvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'])
refine.iterativeBackProjection(
shiftedStack,
evenfile,
rad=self.params['rad'],
ang=outang,
out=evenvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'])
else:
refine.backProjection(
shiftedStack,
oddfile,
ang=outang,
out=oddvol,
sym=sydoc,
nproc=self.params['proc'],
)
refine.backProjection(
shiftedStack,
evenfile,
ang=outang,
out=evenvol,
sym=sydoc,
nproc=self.params['proc'],
)
# calculate the FSC
apDisplay.printMsg("calculating FSC")
fscfile = "fsc%03d.spi" % iter
emanfsc = "fsc.eotest.%d" % iter
refine.calcFSC(oddvol, evenvol, fscfile)
# convert to eman-style fscfile
refine.spiderFSCtoEMAN(fscfile, emanfsc)
# calculate the resolution at 0.5 FSC & write to file
res = apRecon.calcRes(emanfsc, self.stack['boxsize'],
self.stack['apix'])
restxt = "resolution.txt"
if iter == 1 and os.path.isfile(restxt):
os.remove(restxt)
resfile = open(restxt, "a")
resfile.write("iter %d:\t%.3f\n" % (iter, res))
resfile.close()
# filter & normalize the volume to be used as a reference in the next round
outvol = "vol%03d.spi" % iter
emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % (
out_rawvol, outvol, self.stack['apix'], res,
self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# create mrc files of volumes
emancmd = "proc3d %s %s apix=%.3f mask=%d norm" % (
out_rawvol, "threed.%da.mrc" % iter, self.stack['apix'],
self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % (
out_rawvol, "threed.%da.lp.mrc" % iter, self.stack['apix'],
res, self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.param['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# set this model as start for next iteration, remove previous
os.remove(self.params['itervol'])
os.remove(out_rawvol)
self.params['itervol'] = outvol
# clean up directory
apDisplay.printMsg("cleaning up directory")
if os.path.isfile(oddvol):
os.remove(oddvol)
if os.path.isfile(evenvol):
os.remove(evenvol)
if os.path.isfile(ang):
os.remove(ang)
if os.path.isfile(sel):
os.remove(sel)
if os.path.isfile(projs):
os.remove(projs)
if os.path.isfile(oddfile):
os.remove(oddfile)
if os.path.isfile(evenfile):
os.remove(evenfile)
if os.path.isfile(emanfsc) and os.path.isfile(fscfile):
os.remove(fscfile)
if os.path.isfile(shiftedStack):
os.remove(shiftedStack)
os.remove(self.params['itervol'])
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### setup Xmipp command
aligntime = time.time()
xmippopts = (" -i "+os.path.join(self.params['rundir'], self.params['localstack'])
+" --nref "+str(self.params['numrefs'])
+" --iter "+str(self.params['maxiter'])
+" --odir "+str(self.params['rundir'])
+" --oroot "+ "part"+str(self.params['timestamp'])
+" --classifyAllImages"
)
if self.params['correlation']:
xmippopts += " --distance correlation"
if self.params['classical']:
xmippopts += " --classicalMultiref"
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath(self.execFile, die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp 3", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### post-processing
# Create a stack for the class averages at each level
Nlevels=glob.glob("level_*")
for level in Nlevels:
digits = level.split("_")[1]
apParam.runCmd("xmipp_image_convert -i "+level+"/part"+self.params['timestamp']+"*xmd -o part"
+self.params['timestamp']+"_level_"+digits+"_.hed", package="Xmipp 3", verbose=True)
if self.params['align']:
apParam.runCmd("xmipp_transform_geometry -i images.xmd -o %s_aligned.stk --apply_transform" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apParam.runCmd("xmipp_image_convert -i %s_aligned.xmd -o alignedStack.hed" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apFile.removeFile("%s_aligned.xmd" % self.params['timestamp'])
apFile.removeFile("%s_aligned.stk" % self.params['timestamp'])
self.parseOutput()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
self.clearIntermediateFiles()
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.clipsize = int(math.floor(self.stack['boxsize']/self.params['bin']/2.0)*self.params['bin']*2.0)
#self.checkRunNamePath()
### convert stack to spider
spiderstack = self.createSpiderFile()
#create template stack
templatestack = self.createTemplateStack()
#create orientation reference
orientref = self.createOrientationReference()
###################################################################
aligntime = time.time()
### create batch file
batchfilepath = self.setupBatchFile(spiderstack, templatestack, orientref)
### run the spider alignment
apDisplay.printColor("Running iterative ref-classification and free-alignment with spider","cyan")
self.runSpiderBatch( batchfilepath, spiderstack )
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
###################################################################
### remove unaligned spider stack
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=False)
### check to be sure files exist
avgfile = os.path.join(self.params['rundir'], "alignstack.spi") #class averages
if not os.path.isfile(avgfile):
apDisplay.printError("Final stack of aligned particles does not exist.")
### convert stacks to imagic
self.convertStack2Imagic("alignstack.spi")
self.convertStack2Imagic("avg.spi")
### make alignment average in mrc format
emancmd = "proc2d avg.spi average.mrc average"
apEMAN.executeEmanCmd(emancmd)
inserttime = time.time()
if self.params['commit'] is True:
apDisplay.printWarning("committing results to DB")
self.runtime = aligntime
self.insertEdIterRun(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def start(self):
self.runtime = 0
self.partlist = []
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
self.clipsize = int(
math.floor(self.stack['boxsize'] / self.params['bin'] / 2.0) *
self.params['bin'] * 2.0)
#self.checkRunNamePath()
### convert stack to spider
spiderstack = self.createSpiderFile()
#create template stack
templatestack = self.createTemplateStack()
#create orientation reference
orientref = self.createOrientationReference()
###################################################################
aligntime = time.time()
### create batch file
batchfilepath = self.setupBatchFile(spiderstack, templatestack,
orientref)
### run the spider alignment
apDisplay.printColor(
"Running iterative ref-classification and free-alignment with spider",
"cyan")
self.runSpiderBatch(batchfilepath, spiderstack)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
###################################################################
### remove unaligned spider stack
apDisplay.printMsg("Removing un-aligned stack: " + spiderstack)
apFile.removeFile(spiderstack, warn=False)
### check to be sure files exist
avgfile = os.path.join(self.params['rundir'],
"alignstack.spi") #class averages
if not os.path.isfile(avgfile):
apDisplay.printError(
"Final stack of aligned particles does not exist.")
### convert stacks to imagic
self.convertStack2Imagic("alignstack.spi")
self.convertStack2Imagic("avg.spi")
### make alignment average in mrc format
emancmd = "proc2d avg.spi average.mrc average"
apEMAN.executeEmanCmd(emancmd)
inserttime = time.time()
if self.params['commit'] is True:
apDisplay.printWarning("committing results to DB")
self.runtime = aligntime
self.insertEdIterRun(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: " +
apDisplay.timeString(inserttime))
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apParam.runCmd(proccmd, "EMAN", verbose=True)
if self.params['numpart'] != apFile.numImagesInStack(
self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -codes " + str(self.params['numrefs']) + " -iter " +
str(self.params['maxiter']) + " -o " + os.path.join(
self.params['rundir'], "part" + self.params['timestamp']))
if self.params['fast']:
xmippopts += " -fast "
if self.params['correlation']:
xmippopts += " -useCorrelation "
if self.params['classical']:
xmippopts += " -classicalMultiref "
if self.params['align']:
xmippopts += " -alignImages "
### use multi-processor command
apDisplay.printColor(
"Using " + str(self.params['nproc']) + " processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True)
mpiruncmd = self.mpirun + " -np " + str(
self.params['nproc']) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd,
package="Xmipp",
verbose=True,
showcmd=True,
logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(self.params['runtime']))
### minor post-processing
self.createReferenceStack()
self.parseOutput()
self.clearIntermediateFiles()
# self.readyUploadFlag()
apParam.dumpParameters(
self.params, "cl2d-" + self.params['timestamp'] + "-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-" +
self.params['timestamp'] +
"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(
self.runparams['stackid']) * self.runparams['bin']
self.Nlevels = len(
glob.glob("part" + self.params['timestamp'] + "_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part" + self.params[
'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels - 1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part" + self.params['timestamp'] +
"_level_%02d_.hed" % level, classnum + 1,
partdict[classnum], len(partdict))
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
### test insert to make sure data is not overwritten
self.params['runtime'] = 0
#self.checkDuplicateRefBasedRun()
### set box size
self.boxsize = int(math.floor(self.stack['boxsize']/self.params['bin']/2.0))*2
### convert stack to spider
spiderstack = self.createSpiderFile()
### create template stack
templatestack = self.createTemplateStack()
### run the alignment
aligntime = time.time()
usestack = spiderstack
oldpartlist = None
for i in range(self.params['numiter']):
iternum = i+1
apDisplay.printColor("\n\nITERATION "+str(iternum), "green")
alignedstack, partlist = self.runAlignmentGPU(
usestack, templatestack, spiderstack,
self.params['xysearch'], self.params['xystep'],
self.params['firstring'], self.params['lastring'],
iternum=iternum, oldpartlist=oldpartlist)
oldpartlist = partlist
usestack = alignedstack
templatestack = self.updateTemplateStack(alignedstack, partlist, iternum)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### remove large, worthless stack
spiderstack = os.path.join(self.params['rundir'], "start.spi")
apDisplay.printMsg("Removing un-aligned stack: "+spiderstack)
apFile.removeFile(spiderstack, warn=True)
### convert aligned stack to imagic
finalspistack = "aligned.spi"
shutil.move(alignedstack, finalspistack)
imagicstack = "aligned.hed"
apFile.removeStack(imagicstack)
emancmd = "proc2d "+finalspistack+" "+imagicstack
apEMAN.executeEmanCmd(emancmd, verbose=True)
### average stack
apStack.averageStack(imagicstack)
### calculate resolution for each reference
apix = self.stack['apix']*self.params['bin']
self.calcResolution(partlist, imagicstack, apix)
if self.params['commit'] is True:
apDisplay.printMsg("committing results to DB")
self.params['runtime'] = aligntime
self.insertRefBasedRun(partlist, imagicstack, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
### remove temporary files
apFile.removeFilePattern("alignments/alignedstack*.spi")
apFile.removeFile(finalspistack)
def convertToRefineStack(self):
'''
The stack is remaked without ctf correction and without invertion (ccd)
'''
newstackroot = os.path.join(self.params['rundir'],os.path.basename(self.stack['file'])[:-4])
if self.params['paramonly'] is True:
print 'newstackroot',newstackroot
self.setFrealignStack(newstackroot)
return
# copy existing refined stack if possible
existing_refine_stack = apStack.getExistingRefineStack(self.stack['data'],'frealign',False,self.params['last'],self.params['bin'],lowpass=self.params['lowpass'],highpass=self.params['highpass'])
if existing_refine_stack:
if existing_refine_stack != os.path.join(self.params['rundir'],newstackroot+'.mrc'):
shutil.copyfile(existing_refine_stack,newstackroot+'.mrc')
new_partorderfile = os.path.join(self.params['rundir'],'stackpartorder.list')
existing_partorderfile = os.path.join(os.path.dirname(existing_refine_stack),'stackpartorder.list')
# particle order list also need to be copied
if not os.path.isfile(new_partorderfile) and os.path.isfile(existing_partorderfile):
shutil.copyfile(existing_partorderfile,new_partorderfile)
self.setFrealignStack(newstackroot)
return
if self.no_ctf_correction:
self.ImagicStackToFrealignMrcStack(self.stack['file'])
self.setFrealignStack(newstackroot)
return
# stack need to be remade without ctf correction
apDisplay.printWarning('Frealign needs a stack without ctf correction. A new stack is being made....')
stackdata = self.stack['data']
madestackdata = self.getOriginalStackMadeByMakeStack()
stackid = stackdata.dbid
stackruns = apStack.getStackRunsFromStack(madestackdata)
# To Do: need to handle combined stack
stackrun = stackruns[0]
stackpathname = os.path.basename(stackdata['path']['path'])
totalpart = apStack.getNumberStackParticlesFromId(stackid)
if not self.params['last']:
numpart = totalpart
else:
numpart = min(self.params['last'],totalpart)
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir,'temp.hed')
# use first particle image to get presetname
oneparticle = apStack.getOneParticleFromStackId(stackid, particlenumber=1)
preset =oneparticle['particle']['image']['preset']
if preset:
presetname = preset['name']
else:
presetname = 'manual'
# use first stack run to get parameters
paramdata = stackrun['stackParams']
# binning is combination of the original binning of the stack and the preparation binnning
bin = paramdata['bin']*self.params['bin']
unbinnedboxsize = self.stack['boxsize']*paramdata['bin']
lowpasstext = self.setArgText('lowpass',(self.params['lowpass'],paramdata['lowpass']),False)
highpasstext = self.setArgText('highpass',(self.params['highpass'],paramdata['highpass']),True)
partlimittext = self.setArgText('partlimit',(numpart,),False)
xmipp_normtext = self.setArgText('xmipp-normalize',(paramdata['xmipp-norm'],),True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
# The assumption is that the image is from ice grid and digital camera (black particles on white background
if 'reverse' in self.stackrunlogparams.keys():
reversetext = '--reverse'
else:
reversetext = ''
if 'defocpair' in self.stackrunlogparams.keys():
defoctext = '--defocpair'
else:
defoctext = ''
cmd = '''
makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s --no-invert --normalized %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2
''' % (os.path.basename(newstackimagicfile),stackid,lowpasstext,highpasstext,partlimittext,reversetext,defoctext,xmipp_normtext,unbinnedboxsize,bin,stackpathname,stackid,projectid,presetname,newstackrunname,newstackrundir,sessionname,sessionid)
logfilepath = os.path.join(newstackrundir,'frealignstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd,logfilepath)
if returncode > 0:
apDisplay.printError('Error in Frealign specific stack making')
self.ImagicStackToFrealignMrcStack(newstackimagicfile)
os.rename(newstackimagicfile[:-4]+'.mrc',newstackroot+'.mrc')
self.setFrealignStack(newstackroot)
# use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
#clean up
rmfiles = glob.glob("*.box")
for rmfile in rmfiles:
apFile.removeFile(rmfile)
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
# symmetry info
if self.params['sym']=='Icos':
self.params['symtype']='I'
self.params['symfold']=None
else:
self.params['symtype']=self.params['sym'][0]
self.params['symfold']=int(self.params['sym'][1:])
# eman "d" symmetry is spider "ci"
if self.params['symtype'].upper()=="D":
self.params['symtype'] = "CI"
# create symmetry doc file
sydoc="sym.spi"
refine.symmetryDoc(self.params['symtype'],self.params['symfold'],sydoc)
# convert incr to array of increments
ang_inc=self.params['incr'].split(',')
self.params['numiter'] = len(ang_inc)
self.params['increments']=[]
for i in range(0,self.params['numiter']):
self.params['increments'].append(int(ang_inc[i]))
# convert stack to spider stack
spiderstack=os.path.join(self.params['rundir'],'start.spi')
operations.stackToSpiderStack(
self.stack['file'],
spiderstack,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
numpart=self.params['numpart']
)
# create filtered stack
spiderstackfilt=spiderstack
if (self.params['lowpass']+self.params['highpass']) > 0:
spiderstackfilt=os.path.join(self.params['rundir'],'start_filt.spi')
operations.stackToSpiderStack(
self.stack['file'],
spiderstackfilt,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
lp=self.params['lowpass'],
hp=self.params['highpass'],
numpart=self.params['numpart']
)
# rescale initial model if necessary
outvol = os.path.join(self.params['rundir'],"vol000.spi")
apModel.rescaleModel(self.params['modelid'], outvol, self.stack['boxsize'], self.stack['apix'], spider=True)
self.params['itervol']=outvol
for iter in range(1,self.params['numiter']+1):
# create projections for projection matching
apDisplay.printMsg("creating reference projections of volume: %s" % self.params['itervol'])
projs,numprojs,ang,sel = refine.createProjections(
incr=self.params['increments'][iter-1],
boxsz=self.stack['boxsize'],
symfold=self.params['symfold'],
invol=self.params['itervol'],
rad=self.params['rad'],
)
# run reference-based alignment
apDisplay.printMsg("running reference-based alignment (AP MQ)")
apmqfile = "apmq%03d.spi" % iter
outang = "angular%03d.spi" % iter
shf = "shifts%03d.spi" % iter
shiftedStack="parts_shifted.spi"
apFile.removeFile(shf)
refine.spiderAPMQ(
projs=projs,
numprojs=numprojs,
tsearch=self.params['xysearch'],
tstep=self.params['xystep'],
firstRing=self.params['firstring'],
lastRing=self.params['lastring'],
stackfile=spiderstackfilt,
nump=self.params['numpart'],
ang=ang,
apmqfile=apmqfile,
outang=outang,
nproc=self.params['proc'],
)
# use cross-correlation to find the sub-pixel alignment
# of the particles,
# results will be saved in "peakfile.spi"
apFile.removeFile(shiftedStack)
# don't use MPI here - for some reason slower?
mySpi=spyder.SpiderSession(dataext=".spi", logo=False, log=False)
apmqlist = refine.readDocFile(apmqfile)
avgccrot = 0
apDisplay.printMsg("creating shifted stack")
for p in range(0,self.params['numpart']):
ref=int(float(apmqlist[p][2]))
ccrot=float(apmqlist[p][3])
inplane=float(apmqlist[p][4])
avgccrot+=ccrot
# invert the sign - ref projs will be rotated
inplane*=-1
# get corresponding projection
if (ref <= 0):
# mirror projection if necessary
ref*=-1
refimg=spyder.fileFilter(projs)+"@"+str(ref)
refine.mirrorImg(refimg,"_3",inMySpi=mySpi)
img="_3"
else:
img=spyder.fileFilter(projs)+"@"+str(ref)
refine.rotAndShiftImg(img,"_2",inplane,inMySpi=mySpi)
refine.maskImg("_2","_3",self.params['rad'],"D","E",
center=int((self.stack['boxsize']/2)+1),
inMySpi=mySpi)
# pad ref image & stack image to twice the size
refine.padImg("_3","_2",2*self.stack['boxsize'],"N",1,1,0,inMySpi=mySpi)
stackimg=spyder.fileFilter(spiderstack)+"@"+str(p+1)
refine.padImg(stackimg,"_1",2*self.stack['boxsize'],"B",1,1,inMySpi=mySpi)
# calculate cross-correlation
refine.getCC("_1","_2","_1",inMySpi=mySpi)
# crop the correllation image to allowable shift amount
shift=int(self.params['allowedShift']*self.stack['boxsize'])
dim=2*shift+1
topleftx=self.stack['boxsize']-shift+1
refine.windowImg("_1","_2",dim,topleftx,topleftx,inMySpi=mySpi)
# find the sub-pixel location of cc peak
mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17","_2","0")
# create new stack of shifted particles
shpos=spyder.fileFilter(shiftedStack)+"@"+str(p+1)
mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN")
mySpi.toSpiderQuiet("GP x17","_2",str(shift+1)+","+str(shift+1))
refine.copyImg(stackimg,shpos,inMySpi=mySpi)
mySpi.toSpiderQuiet("ELSE")
#mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16")
mySpi.toSpiderQuiet("SH F",stackimg,shpos,"-x15,-x16")
mySpi.toSpiderQuiet("ENDIF")
# save shifts to file
mySpi.toSpiderQuiet("SD "+str(p+1)+",x15,x16,x17",spyder.fileFilter(shf))
mySpi.toSpiderQuiet("SD E",spyder.fileFilter(shf))
mySpi.close()
# create class average images
refine.createClassAverages(
shiftedStack,
projs,
apmqfile,
numprojs,
self.stack['boxsize'],
shifted=True,
)
# rename class averages & variacnes for iteration
cmd="/bin/mv classes.hed classes.%d.hed;" % iter
cmd+="/bin/mv classes.img classes.%d.img;" % iter
cmd+="/bin/mv variances.hed variances.%d.hed;" % iter
cmd+="/bin/mv variances.img variances.%d.img;" % iter
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
# calculate the stddev for the apmq cc's for throwing out particles
avgccrot/=self.params['numpart']
stdccrot = 0
for p in range(0,self.params['numpart']):
stdccrot+=abs(float(apmqlist[p][3])-avgccrot)
stdccrot/=self.params['numpart']
cccutoff=avgccrot+(stdccrot*self.params['keepsig'])
apDisplay.printMsg("average cc: %f" %avgccrot)
apDisplay.printMsg("setting cutoff to: %f" %cccutoff)
# create new selection file that only has particles with good cc's
selectfile="select%03d.spi" % iter
apFile.removeFile(selectfile)
mySpi = spyder.SpiderSession(nproc=self.params['proc'],dataext=".spi", logo=False, log=False)
i=1
for p in range(0,self.params['numpart']):
ccrot=float(apmqlist[p][3])
if ccrot>=cccutoff:
mySpi.toSpiderQuiet("x11=%d" % (p+1))
mySpi.toSpiderQuiet("SD %d,x11" % i,spyder.fileFilter(selectfile))
i+=1
mySpi.close()
# calculate the new 3d structure using centered projections
# and the corrected angles from the angular doc file
apDisplay.printMsg("creating 3d volume")
out_rawvol="vol_raw%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
selectfile,
rad=self.params['rad'],
ang=outang,
out=out_rawvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['bpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'],
)
else:
refine.backProjection(
shiftedStack,
selectfile,
ang=outang,
out=out_rawvol,
sym=sydoc,
nproc=self.params['proc']
)
# create even & odd select files
apDisplay.printMsg("creating even/odd volumes")
oddfile="selectodd%03d.spi" % iter
evenfile="selecteven%03d.spi" % iter
refine.docSplit(selectfile,oddfile,evenfile)
# get the even & odd volumesa
oddvol="vol1%03d.spi" % iter
evenvol="vol2%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
oddfile,
rad=self.params['rad'],
ang=outang,
out=oddvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc']
)
refine.iterativeBackProjection(
shiftedStack,
evenfile,
rad=self.params['rad'],
ang=outang,
out=evenvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc']
)
else:
refine.backProjection(
shiftedStack,
oddfile,
ang=outang,
out=oddvol,
sym=sydoc,
nproc=self.params['proc'],
)
refine.backProjection(
shiftedStack,
evenfile,
ang=outang,
out=evenvol,
sym=sydoc,
nproc=self.params['proc'],
)
# calculate the FSC
apDisplay.printMsg("calculating FSC")
fscfile="fsc%03d.spi" % iter
emanfsc="fsc.eotest.%d" % iter
refine.calcFSC(oddvol,evenvol,fscfile)
# convert to eman-style fscfile
refine.spiderFSCtoEMAN(fscfile,emanfsc)
# calculate the resolution at 0.5 FSC & write to file
res=apRecon.calcRes(emanfsc,self.stack['boxsize'],self.stack['apix'])
restxt="resolution.txt"
if iter==1 and os.path.isfile(restxt):
os.remove(restxt)
resfile=open(restxt,"a")
resfile.write("iter %d:\t%.3f\n" % (iter,res))
resfile.close()
# filter & normalize the volume to be used as a reference in the next round
outvol="vol%03d.spi" % iter
emancmd="proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % (out_rawvol,outvol,self.stack['apix'],res,self.params['rad'])
if self.params['imask'] is not None:
emancmd+=" imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# create mrc files of volumes
emancmd="proc3d %s %s apix=%.3f mask=%d norm" % (out_rawvol,"threed.%da.mrc" % iter, self.stack['apix'],self.params['rad'])
if self.params['imask'] is not None:
emancmd+=" imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
emancmd="proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % (out_rawvol,"threed.%da.lp.mrc" % iter, self.stack['apix'],res, self.params['rad'])
if self.params['imask'] is not None:
emancmd+=" imask=%d" % self.param['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# set this model as start for next iteration, remove previous
os.remove(self.params['itervol'])
os.remove(out_rawvol)
self.params['itervol']=outvol
# clean up directory
apDisplay.printMsg("cleaning up directory")
if os.path.isfile(oddvol):
os.remove(oddvol)
if os.path.isfile(evenvol):
os.remove(evenvol)
if os.path.isfile(ang):
os.remove(ang)
if os.path.isfile(sel):
os.remove(sel)
if os.path.isfile(projs):
os.remove(projs)
if os.path.isfile(oddfile):
os.remove(oddfile)
if os.path.isfile(evenfile):
os.remove(evenfile)
if os.path.isfile(emanfsc) and os.path.isfile(fscfile):
os.remove(fscfile)
if os.path.isfile(shiftedStack):
os.remove(shiftedStack)
os.remove(self.params['itervol'])
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -codes "+str(self.params['numrefs'])
+" -iter "+str(self.params['maxiter'])
+" -o "+os.path.join(self.params['rundir'], "part"+self.params['timestamp'])
)
if self.params['fast']:
xmippopts += " -fast "
if self.params['correlation']:
xmippopts += " -useCorrelation "
if self.params['classical']:
xmippopts += " -classicalMultiref "
if self.params['align']:
xmippopts += " -alignImages "
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### minor post-processing
self.createReferenceStack()
self.parseOutput()
self.clearIntermediateFiles()
# self.readyUploadFlag()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
def checkConflicts(self):
### first get all stack data
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### modify boxsize and pixelsize according to binning
self.boxsize = self.stack['boxsize']
self.clipsize = int(
math.floor(self.boxsize / float(self.params['bin'] * 2))) * 2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d" %
(self.params['clipsize'], self.clipsize))
self.clipsize = self.params['clipsize']
self.apix = self.stack['apix']
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
self.boxsize = clipsize
self.apix = self.apix * self.params['bin']
### basic error checking
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['ncls'] is None:
apDisplay.printError("a number of classes was not provided")
maxparticles = 1000000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
if self.params['numpart'] > apFile.numImagesInStack(stackfile):
apDisplay.printError("trying to use more particles " +
str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(stackfile)))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > 5000:
apDisplay.printWarning(
"initial model calculation may not work with less than 5000 particles"
)
self.mpirun = self.checkMPI()
# if self.mpirun is None:
# apDisplay.printError("There is no MPI installed")
if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
### SIMPLE defaults and error checking
if self.params['ring2'] is None:
self.params['ring2'] = (self.boxsize / 2) - 2
if self.params['ncls'] > 2000:
apDisplay.printError(
"number of classes should be less than 2000 for subsequent ORIGAMI run to work"
)
if self.params['ncls'] > self.params['numpart']:
self.params['ncls'] = self.params['numpart'] / self.params['minp']
if self.params['mask'] is None:
self.params['mask'] = (self.boxsize / 2) - 2
if self.params['mw'] is None:
apDisplay.printError(
"please specify the molecular weight (in kDa)")
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local temp file
proccmd = "proc2d "+self.stack['file']+" temp.hed apix="+str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize)*self.params['bin']
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
proccmd += " last="+str(self.params['numpart']-1)
apEMAN.executeEmanCmd(proccmd, verbose=True)
### process stack to final file
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
proccmd = "proc2d temp.hed "+self.params['localstack']+" apix="+str(self.stack['apix']*self.params['bin'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
apEMAN.executeEmanCmd(proccmd, verbose=True)
apFile.removeStack("temp.hed")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### convert stack into single spider files
templateselfile = self.initializeTemplates()
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -iter "+str(self.params['maxiter'])
+" -ref "+templateselfile
+" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp)
+" -psi_step "+str(self.params['psistep'])
)
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-18 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using "+str(nproc-1)+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun+" -np "+str(nproc-1)+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe+" "+xmippopts
self.writeXmippLog(xmippcmd)
apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()