def getBeamTiltFromStackParticleNumber(self): stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum+1, msg=True) pdata = stackparticledata['particle'] ### step 1: get axial beam tilt on to image coordinate direction imageshift = pdata['image']['scope']['image shift'] zerobeamtilt = {'x':0.0e-3, 'y':0.0e-3} axialbeamtilt = self.transformImageShiftToBeamTilt(imageshift, self.tem, self.cam, self.ht, zerobeamtilt, self.mag) beamtilt_rotationtransform = self.retrieveBeamTiltMatrix(self.tem, self.cam, self.ht, self.mag) axialbtvector = numpy.array((axialbeamtilt['y'],axialbeamtilt['x'])) axialbtvector_on_image = numpy.dot(beamtilt_rotationtransform, axialbtvector) apDisplay.printMsg( "Axial Beam Tilt on Image Coordinate ( %5.2f, %5.2f)" % (axialbtvector_on_image[1]*1e3,axialbtvector_on_image[0]*1e3)) ### step 2: get off-axis beam tilt pcoord = {'x':pdata['xcoord']*self.imgpix,'y':pdata['ycoord']*self.imgpix} apix = apStack.getStackPixelSizeFromStackId(self.stackid) camdata = pdata['image']['camera'] cambin = camdata['binning'] camdim = camdata['dimension'] camoff = camdata['offset'] camcenter = {'x':self.camsize['x']/2,'y':self.camsize['y']/2} apDisplay.printMsg( 'Particle coord (%5d,%5d)' % (pdata['xcoord'],pdata['ycoord'])) mcoord = {'y':pdata['ycoord']*cambin['y']+camoff['y']-camcenter['y'], 'x':pdata['xcoord']*cambin['x']+camoff['x']-camcenter['x']} apDisplay.printMsg( 'Particle from center in camera pixel (%5d,%5d)' % (mcoord['x'],mcoord['y'])) coord = {'x':mcoord['x']*self.imgpix*cambin['x'], 'y':mcoord['y']*self.imgpix*cambin['y']} apDisplay.printMsg( 'Particle from center (A) (%5.2f,%5.2f)' % (coord['x']*1e10,coord['y']*1e10)) offaxisbeamtilt = self.getCombinedOffAxisBeamTilt(self.c2diameter,self.beamdiameter,coord) #offaxisbeamtilt = {'x':0.0,'y':0.0} ### step 3: combine beam tilts totalbeamtilt = {'x':axialbtvector_on_image[1]+offaxisbeamtilt['x'], 'y':axialbtvector_on_image[0]-offaxisbeamtilt['y'] } apDisplay.printMsg( "Total Beam Tilt on Image Coordinate (mrad) ( %5.2f, %5.2f)" % (totalbeamtilt['x']*1e3,totalbeamtilt['y']*1e3)) return totalbeamtilt
def setupParticles(self): self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2 apix = self.stack['apix']*self.params['bin'] proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix']) if self.params['bin'] > 1: clipsize = boxsize*self.params['bin'] proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) if self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) proccmd += " last="+str(self.params['numpart']-1) apFile.removeStack(self.params['localstack'], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) return (boxsize, apix)
def setupParticles(self): self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2 apix = self.stack['apix']*self.params['bin'] proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix']) if self.params['bin'] > 1: clipsize = boxsize*self.params['bin'] proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) if self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) proccmd += " last="+str(self.params['numpart']-1) apFile.removeStack(self.params['localstack'], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) return (boxsize, apix)
def start(self): self.runtime = 0 self.partlist = [] self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.checkNoRefRun() ### convert stack to spider spiderstack = self.createSpiderFile() ### create initialization template if self.params['initmethod'] == 'allaverage': templatefile = self.averageTemplate() elif self.params['initmethod'] == 'selectrand': templatefile = self.selectRandomParticles() elif self.params['initmethod'] == 'randpart': templatefile = self.pickRandomParticle() elif self.params['initmethod'] == 'template': templatefile = self.getTemplate() else: apDisplay.printError("unknown initialization method defined: " +str(self.params['initmethod'])+" not in "+str(self.initmethods)) apDisplay.printColor("Running spider this can take awhile","cyan") ### run the alignment aligntime = time.time() pixrad = int(round(self.params['partrad']/self.stack['apix']/self.params['bin'])) alignedstack, self.partlist = alignment.refFreeAlignParticles( spiderstack, templatefile, self.params['numpart'], pixrad, self.params['firstring'], self.params['lastring'], rundir = ".") aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ### remove large, worthless stack spiderstack = os.path.join(self.params['rundir'], "start.spi") apDisplay.printMsg("Removing un-aligned stack: "+spiderstack) apFile.removeFile(spiderstack, warn=False) ### convert stack to imagic imagicstack = self.convertSpiderStack(alignedstack) apFile.removeFile(alignedstack) inserttime = time.time() if self.params['commit'] is True: self.runtime = aligntime self.insertNoRefRun(imagicstack, insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def setupParticles(self): self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed") self.stack = {} self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"]) self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"]) self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"]) self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"]) boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2 apix = self.stack["apix"] * self.params["bin"] proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"]) if self.params["bin"] > 1: clipsize = boxsize * self.params["bin"] proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize) if self.params["highpass"] > 1: proccmd += " hp=" + str(self.params["highpass"]) if self.params["lowpass"] > 1: proccmd += " lp=" + str(self.params["lowpass"]) proccmd += " last=" + str(self.params["numpart"] - 1) apFile.removeStack(self.params["localstack"], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"]) return (boxsize, apix)
def getBeamTiltFromStackParticleNumber(self): stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum + 1, msg=True) pdata = stackparticledata['particle'] ### step 1: get axial beam tilt on to image coordinate direction imageshift = pdata['image']['scope']['image shift'] zerobeamtilt = {'x': 0.0e-3, 'y': 0.0e-3} axialbeamtilt = self.transformImageShiftToBeamTilt( imageshift, self.tem, self.cam, self.ht, zerobeamtilt, self.mag) beamtilt_rotationtransform = self.retrieveBeamTiltMatrix( self.tem, self.cam, self.ht, self.mag) axialbtvector = numpy.array((axialbeamtilt['y'], axialbeamtilt['x'])) axialbtvector_on_image = numpy.dot(beamtilt_rotationtransform, axialbtvector) apDisplay.printMsg( "Axial Beam Tilt on Image Coordinate ( %5.2f, %5.2f)" % (axialbtvector_on_image[1] * 1e3, axialbtvector_on_image[0] * 1e3)) ### step 2: get off-axis beam tilt pcoord = { 'x': pdata['xcoord'] * self.imgpix, 'y': pdata['ycoord'] * self.imgpix } apix = apStack.getStackPixelSizeFromStackId(self.stackid) camdata = pdata['image']['camera'] cambin = camdata['binning'] camdim = camdata['dimension'] camoff = camdata['offset'] camcenter = {'x': self.camsize['x'] / 2, 'y': self.camsize['y'] / 2} apDisplay.printMsg('Particle coord (%5d,%5d)' % (pdata['xcoord'], pdata['ycoord'])) mcoord = { 'y': pdata['ycoord'] * cambin['y'] + camoff['y'] - camcenter['y'], 'x': pdata['xcoord'] * cambin['x'] + camoff['x'] - camcenter['x'] } apDisplay.printMsg('Particle from center in camera pixel (%5d,%5d)' % (mcoord['x'], mcoord['y'])) coord = { 'x': mcoord['x'] * self.imgpix * cambin['x'], 'y': mcoord['y'] * self.imgpix * cambin['y'] } apDisplay.printMsg('Particle from center (A) (%5.2f,%5.2f)' % (coord['x'] * 1e10, coord['y'] * 1e10)) offaxisbeamtilt = self.getCombinedOffAxisBeamTilt( self.c2diameter, self.beamdiameter, coord) #offaxisbeamtilt = {'x':0.0,'y':0.0} ### step 3: combine beam tilts totalbeamtilt = { 'x': axialbtvector_on_image[1] + offaxisbeamtilt['x'], 'y': axialbtvector_on_image[0] - offaxisbeamtilt['y'] } apDisplay.printMsg( "Total Beam Tilt on Image Coordinate (mrad) ( %5.2f, %5.2f)" % (totalbeamtilt['x'] * 1e3, totalbeamtilt['y'] * 1e3)) return totalbeamtilt
def getInstrumentParams(self): self.stackpix = apStack.getStackPixelSizeFromStackId(self.stackid, True) * 1e-10 stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum+1, msg=True) pdata = stackparticledata['particle'] self.imgpix = apDatabase.getPixelSize(pdata['image'])*1e-10 scopedata = pdata['image']['scope'] self.tem = scopedata['tem'] self.cam = pdata['image']['camera']['ccdcamera'] self.ht = scopedata['high tension'] self.Cs = 2e-3 self.mag = scopedata['magnification'] self.camsize = {'x':4096,'y':4096} self.beamdiameter = 1e-6 self.c2diameter = 1.0e-4 self.wavelength = fftfun.getElectronWavelength(self.ht)
def getInstrumentParams(self): self.stackpix = apStack.getStackPixelSizeFromStackId( self.stackid, True) * 1e-10 stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum + 1, msg=True) pdata = stackparticledata['particle'] self.imgpix = apDatabase.getPixelSize(pdata['image']) * 1e-10 scopedata = pdata['image']['scope'] self.tem = scopedata['tem'] self.cam = pdata['image']['camera']['ccdcamera'] self.ht = scopedata['high tension'] self.Cs = 2e-3 self.mag = scopedata['magnification'] self.camsize = {'x': 4096, 'y': 4096} self.beamdiameter = 1e-6 self.c2diameter = 1.0e-4 self.wavelength = fftfun.getElectronWavelength(self.ht)
def createFastFreeHandInputLineTemplate(self): self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False) self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.apix = 6.02 one_particledata = apStack.getOneParticleFromStackId(self.params['stackid'], particlenumber=1, msg=True) scopedata = one_particledata['particle']['image']['scope'] ht_kv = scopedata['high tension'] / 1000.0 cs_mm = scopedata['tem']['cs'] * 1000 angSearch = self.params['angSearch'] constant_inputlines = [ '%.3f,%.3f,%2f,%.f' %(self.apix,self.params['snr'],cs_mm,ht_kv) ,'1,0' ,'../%s' %(self.stackfile) ,'../%s' %(self.modelfile) ,'../%s' %(self.frealign_paramfile) ,'out.mrc' ,'-%d,%d,-%d,%d' %(angSearch,angSearch,angSearch,angSearch) ,'%.1f,%.2f,%.2f' %(self.params['minres'],self.params['maxres'],self.params['radius']) ,'%s' %(self.params['scoringtype'].upper()) ] return constant_inputlines
def createFastFreeHandInputLineTemplate(self): self.boxsize = apStack.getStackBoxsize(self.params["stackid"], msg=False) self.apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) self.apix = 6.02 one_particledata = apStack.getOneParticleFromStackId(self.params["stackid"], particlenumber=1, msg=True) scopedata = one_particledata["particle"]["image"]["scope"] ht_kv = scopedata["high tension"] / 1000.0 cs_mm = scopedata["tem"]["cs"] * 1000 angSearch = self.params["angSearch"] constant_inputlines = [ "%.3f,%.3f,%2f,%.f" % (self.apix, self.params["snr"], cs_mm, ht_kv), "1,0", "../%s" % (self.stackfile), "../%s" % (self.modelfile), "../%s" % (self.frealign_paramfile), "out.mrc", "-%d,%d,-%d,%d" % (angSearch, angSearch, angSearch, angSearch), "%.1f,%.2f,%.2f" % (self.params["minres"], self.params["maxres"], self.params["radius"]), "%s" % (self.params["scoringtype"].upper()), ] return constant_inputlines
def checkConflicts(self): ### first get all stack data self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### modify boxsize and pixelsize according to binning self.boxsize = self.stack['boxsize'] self.clipsize = int(math.floor(self.boxsize/float(self.params['bin']*2)))*2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" %(self.params['clipsize'],self.clipsize)) self.clipsize = self.params['clipsize'] self.apix = self.stack['apix'] if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize)*self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even self.boxsize = clipsize self.apix = self.apix * self.params['bin'] ### basic error checking if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['ncls'] is None: apDisplay.printError("a number of classes was not provided") maxparticles = 1000000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) if self.params['numpart'] > 5000: apDisplay.printWarning("initial model calculation may not work with less than 5000 particles") self.mpirun = self.checkMPI() # if self.mpirun is None: # apDisplay.printError("There is no MPI installed") if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() ### SIMPLE defaults and error checking if self.params['ring2'] is None: self.params['ring2'] = (self.boxsize/2) - 2 if self.params['ncls'] > 2000: apDisplay.printError("number of classes should be less than 2000 for subsequent ORIGAMI run to work") if self.params['ncls'] > self.params['numpart']: self.params['ncls'] = self.params['numpart'] / self.params['minp'] if self.params['mask'] is None: self.params['mask'] = (self.boxsize/2) - 2 if self.params['mw'] is None: apDisplay.printError("please specify the molecular weight (in kDa)")
def start(self): self.runtime = 0 self.partlist = [] self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.checkNoRefRun() ### convert stack to spider spiderstack = self.createSpiderFile() ### create initialization template if self.params['initmethod'] == 'allaverage': templatefile = self.averageTemplate() elif self.params['initmethod'] == 'selectrand': templatefile = self.selectRandomParticles() elif self.params['initmethod'] == 'randpart': templatefile = self.pickRandomParticle() elif self.params['initmethod'] == 'template': templatefile = self.getTemplate() else: apDisplay.printError("unknown initialization method defined: " + str(self.params['initmethod']) + " not in " + str(self.initmethods)) apDisplay.printColor("Running spider this can take awhile", "cyan") ### run the alignment aligntime = time.time() pixrad = int( round(self.params['partrad'] / self.stack['apix'] / self.params['bin'])) alignedstack, self.partlist = alignment.refFreeAlignParticles( spiderstack, templatefile, self.params['numpart'], pixrad, self.params['firstring'], self.params['lastring'], rundir=".") aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### remove large, worthless stack spiderstack = os.path.join(self.params['rundir'], "start.spi") apDisplay.printMsg("Removing un-aligned stack: " + spiderstack) apFile.removeFile(spiderstack, warn=False) ### convert stack to imagic imagicstack = self.convertSpiderStack(alignedstack) apFile.removeFile(alignedstack) inserttime = time.time() if self.params['commit'] is True: self.runtime = aligntime self.insertNoRefRun(imagicstack, insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) apDisplay.printMsg("Database Insertion time: " + apDisplay.timeString(inserttime))
def getSessionDataFromReconId(reconid): stackid = apStack.getStackIdFromRecon(reconid) partdata = apStack.getOneParticleFromStackId(stackid, msg=False) sessiondata = partdata['particle']['selectionrun']['session'] return sessiondata
def start(self): self.addToLog('.... Setting up new ISAC job ....') self.addToLog('.... Making command for stack pre-processing ....') self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### send file to remotehost tasks = {} sfhed = self.stack['file'][:-4] + ".hed" sfimg = self.stack['file'][:-4] + ".img" tasks = self.addToTasks( tasks, "rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s" % (sfhed, self.params['localhost'], self.params['remoterundir'], "start1.hed")) tasks = self.addToTasks( tasks, "rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s" % (sfimg, self.params['localhost'], self.params['remoterundir'], "start1.img")) ### write Sparx jobfile: process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join( self.params['rundir'], self.params['timestamp'] + ".hed") if os.path.isfile(self.params['localstack']): apFile.removeStack(self.params['localstack']) proccmd = "proc2d " + self.stack['file'] + " " + self.params[ 'localstack'] + " apix=" + str(self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], clipsize, clipsize) proccmd += " last=" + str(self.params['numpart'] - 1) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp=" + str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp=" + str(self.params['lowpass']) # apParam.runCmd(proccmd, "EMAN", verbose=True) self.addSimpleCommand('cd %s' % self.params['rundir']) self.addSimpleCommand(proccmd) sparxcmd = "sxcpy.py %s %s_1.hdf" % (self.params['localstack'], self.params['localstack'][:-4]) # apParam.runCmd(sparxcmd, "SPARX", verbose=True) self.addSimpleCommand(sparxcmd) self.addSimpleCommand("") ### write Sparx jobfile: run ISAC for i in range(self.params['generations']): sparxopts = " %s_%d.hdf" % (os.path.join( self.params['localstack'][:-4]), (i + 1)) if self.params['ir'] is not None: sparxopts += " --ir %d" % int( float( apRecon.getComponentFromVector( self.params['ir'], i, splitter=":"))) if self.params['ou'] is not None: sparxopts += " --ou %d" % int( float( apRecon.getComponentFromVector( self.params['ou'], i, splitter=":"))) if self.params['rs'] is not None: sparxopts += " --rs %d" % int( float( apRecon.getComponentFromVector( self.params['rs'], i, splitter=":"))) if self.params['ts'] is not None: sparxopts += " --ts %.1f" % int( float( apRecon.getComponentFromVector( self.params['ts'], i, splitter=":"))) if self.params['xr'] is not None: sparxopts += " --xr %.1f" % int( float( apRecon.getComponentFromVector( self.params['xr'], i, splitter=":"))) if self.params['yr'] is not None: sparxopts += " --yr %.1f" % int( float( apRecon.getComponentFromVector( self.params['yr'], i, splitter=":"))) if self.params['maxit'] is not None: sparxopts += " --maxit %d" % int( float( apRecon.getComponentFromVector( self.params['maxit'], i, splitter=":"))) if self.params['FL'] is not None: sparxopts += " --FL %.2f" % int( float( apRecon.getComponentFromVector( self.params['FL'], i, splitter=":"))) if self.params['FH'] is not None: sparxopts += " --FH %.2f" % int( float( apRecon.getComponentFromVector( self.params['FH'], i, splitter=":"))) if self.params['FF'] is not None: sparxopts += " --FF %.2f" % int( float( apRecon.getComponentFromVector( self.params['FF'], i, splitter=":"))) if self.params['init_iter'] is not None: sparxopts += " --init_iter %d" % int( float( apRecon.getComponentFromVector( self.params['init_iter'], i, splitter=":"))) if self.params['main_iter'] is not None: sparxopts += " --main_iter %d" % int( float( apRecon.getComponentFromVector( self.params['main_iter'], i, splitter=":"))) if self.params['iter_reali'] is not None: sparxopts += " --iter_reali %d" % int( float( apRecon.getComponentFromVector( self.params['iter_reali'], i, splitter=":"))) if self.params['match_first'] is not None: sparxopts += " --match_first %d" % int( float( apRecon.getComponentFromVector( self.params['match_first'], i, splitter=":"))) if self.params['max_round'] is not None: sparxopts += " --max_round %d" % int( float( apRecon.getComponentFromVector( self.params['max_round'], i, splitter=":"))) if self.params['match_second'] is not None: sparxopts += " --match_second %d" % int( float( apRecon.getComponentFromVector( self.params['match_second'], i, splitter=":"))) if self.params['stab_ali'] is not None: sparxopts += " --stab_ali %d" % int( float( apRecon.getComponentFromVector( self.params['stab_ali'], i, splitter=":"))) if self.params['thld_err'] is not None: sparxopts += " --thld_err %.2f" % int( float( apRecon.getComponentFromVector( self.params['thld_err'], i, splitter=":"))) if self.params['indep_run'] is not None: sparxopts += " --indep_run %d" % int( float( apRecon.getComponentFromVector( self.params['indep_run'], i, splitter=":"))) if self.params['thld_grp'] is not None: sparxopts += " --thld_grp %d" % int( float( apRecon.getComponentFromVector( self.params['thld_grp'], i, splitter=":"))) if self.params['img_per_grp'] is not None: sparxopts += " --img_per_grp %d" % int( apRecon.getComponentFromVector(self.params['img_per_grp'], i)) sparxopts += " --generation %d" % (i + 1) sparxexe = apParam.getExecPath("sxisac.py", die=True) mpiruncmd = self.mpirun + " -np " + str( self.params['nproc']) + " " + sparxexe + " " + sparxopts bn = os.path.basename(self.params['localstack'])[:-4] e2cmd = "e2proc2d.py %s_%d.hdf %s_%d.hdf --list=\"generation_%d_unaccounted.txt\"" % \ (bn, i+1, bn, i+2, i+1) self.addSimpleCommand(mpiruncmd) self.addSimpleCommand(e2cmd) # print self.tasks # print self.command_list # self.writeCommandListToFile() apParam.dumpParameters( self.params, "isac-" + self.params['timestamp'] + "-params.pickle")
def start(self): self.insertMaxLikeJob() self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.estimateIterTime() self.dumpParameters() ### process stack to local temp file proccmd = "proc2d " + self.stack['file'] + " temp.hed apix=" + str( self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], clipsize, clipsize) proccmd += " last=" + str(self.params['numpart'] - 1) apEMAN.executeEmanCmd(proccmd, verbose=True) ### process stack to final file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp + ".hed") proccmd = "proc2d temp.hed " + self.params[ 'localstack'] + " apix=" + str( self.stack['apix'] * self.params['bin']) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp=" + str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp=" + str(self.params['lowpass']) apEMAN.executeEmanCmd(proccmd, verbose=True) apFile.removeStack("temp.hed") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles( self.params['localstack']) ### convert stack into single spider files templateselfile = self.initializeTemplates() ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params['rundir'], self.partlistdocfile) + " -iter " + str(self.params['maxiter']) + " -ref " + templateselfile + " -o " + os.path.join(self.params['rundir'], "part" + self.timestamp) + " -psi_step " + str(self.params['psistep'])) ### fast mode if self.params['fast'] is True: xmippopts += " -fast " if self.params['fastmode'] == "narrow": xmippopts += " -C 1e-10 " elif self.params['fastmode'] == "wide": xmippopts += " -C 1e-18 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " ### normalization if self.params['norm'] is True: xmippopts += " -norm " ### find number of processors if self.params['nproc'] is None: nproc = nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] mpirun = self.checkMPI() if nproc > 2 and mpirun is not None: ### use multi-processor apDisplay.printColor("Using " + str(nproc - 1) + " processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True) mpiruncmd = mpirun + " -np " + str( nproc - 1) + " " + xmippexe + " " + xmippopts self.writeXmippLog(mpiruncmd) apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True) else: ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe + " " + xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### minor post-processing self.createReferenceStack() self.readyUploadFlag() self.dumpParameters()
def start(self): self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### test insert to make sure data is not overwritten self.params['runtime'] = 0 #self.checkDuplicateRefBasedRun() ### set box size self.boxsize = int(math.floor(self.stack['boxsize']/self.params['bin']/2.0))*2 ### convert stack to spider spiderstack = self.createSpiderFile() ### create template stack templatestack = self.createTemplateStack() ### run the alignment aligntime = time.time() usestack = spiderstack oldpartlist = None for i in range(self.params['numiter']): iternum = i+1 apDisplay.printColor("\n\nITERATION "+str(iternum), "green") alignedstack, partlist = self.runAlignmentGPU( usestack, templatestack, spiderstack, self.params['xysearch'], self.params['xystep'], self.params['firstring'], self.params['lastring'], iternum=iternum, oldpartlist=oldpartlist) oldpartlist = partlist usestack = alignedstack templatestack = self.updateTemplateStack(alignedstack, partlist, iternum) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ### remove large, worthless stack spiderstack = os.path.join(self.params['rundir'], "start.spi") apDisplay.printMsg("Removing un-aligned stack: "+spiderstack) apFile.removeFile(spiderstack, warn=True) ### convert aligned stack to imagic finalspistack = "aligned.spi" shutil.move(alignedstack, finalspistack) imagicstack = "aligned.hed" apFile.removeStack(imagicstack) emancmd = "proc2d "+finalspistack+" "+imagicstack apEMAN.executeEmanCmd(emancmd, verbose=True) ### average stack apStack.averageStack(imagicstack) ### calculate resolution for each reference apix = self.stack['apix']*self.params['bin'] self.calcResolution(partlist, imagicstack, apix) if self.params['commit'] is True: apDisplay.printMsg("committing results to DB") self.params['runtime'] = aligntime self.insertRefBasedRun(partlist, imagicstack, insert=True) else: apDisplay.printWarning("not committing results to DB") ### remove temporary files apFile.removeFilePattern("alignments/alignedstack*.spi") apFile.removeFile(finalspistack)
def convertToRefineStack(self): ''' The stack is remaked without ctf correction and without invertion (ccd) ''' newstackroot = os.path.join(self.params['rundir'], os.path.basename(self.stack['file'])[:-4]) if self.params['paramonly'] is True: print 'newstackroot', newstackroot self.setFrealignStack(newstackroot) return # copy existing refined stack if possible existing_refine_stack = apStack.getExistingRefineStack( self.stack['data'], 'frealign', False, self.params['last'], self.params['bin'], lowpass=self.params['lowpass'], highpass=self.params['highpass']) try: if existing_refine_stack: if existing_refine_stack != os.path.join( self.params['rundir'], newstackroot + '.mrc'): shutil.copyfile(existing_refine_stack, newstackroot + '.mrc') new_partorderfile = os.path.join(self.params['rundir'], 'stackpartorder.list') existing_partorderfile = os.path.join( os.path.dirname(existing_refine_stack), 'stackpartorder.list') # particle order list also need to be copied if not os.path.isfile(new_partorderfile) and os.path.isfile( existing_partorderfile): shutil.copyfile(existing_partorderfile, new_partorderfile) self.setFrealignStack(newstackroot) return except: # make one from scratch if there is error apDisplay.printMsg( 'Error copying existing refine stack, will regenerate') if self.no_ctf_correction: self.ImagicStackToFrealignMrcStack(self.stack['file']) self.setFrealignStack(newstackroot) return # stack need to be remade without ctf correction apDisplay.printWarning( 'Frealign needs a stack without ctf correction. A new stack is being made....' ) stackdata = self.stack['data'] madestackdata = self.getOriginalStackMadeByMakeStack() stackid = stackdata.dbid stackruns = apStack.getStackRunsFromStack(madestackdata) # To Do: need to handle combined stack stackrun = stackruns[0] stackpathname = os.path.basename(stackdata['path']['path']) totalpart = apStack.getNumberStackParticlesFromId(stackid) if not self.params['last']: numpart = totalpart else: numpart = min(self.params['last'], totalpart) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir, 'temp.hed') # use first particle image to get presetname oneparticle = apStack.getOneParticleFromStackId(stackid, particlenumber=1) preset = oneparticle['particle']['image']['preset'] if preset: presetname = preset['name'] else: presetname = 'manual' # use first stack run to get parameters paramdata = stackrun['stackParams'] # binning is combination of the original binning of the stack and the preparation binnning bin = paramdata['bin'] * self.params['bin'] unbinnedboxsize = self.stack['boxsize'] * paramdata['bin'] lowpasstext = self.setArgText( 'lowpass', (self.params['lowpass'], paramdata['lowpass']), False) highpasstext = self.setArgText( 'highpass', (self.params['highpass'], paramdata['highpass']), True) partlimittext = self.setArgText('partlimit', (numpart, ), False) #normalization normtext = '' if 'xmipp-norm' in paramdata.keys(): # myami-3.1 or before normtext = self.setArgText('xmipp-normalize', (paramdata['xmipp-norm'], ), True) normtext = '--normalized ' + normtext elif 'normalizemethod' in paramdata.keys(): # myami-3.2 or newer normtext = '--normalize-method=%s' % ( paramdata['normalizemethod'], ) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] # The assumption is that the image is from ice grid and digital camera (black particles on white background if 'reverse' in self.stackrunlogparams.keys(): reversetext = '--reverse' else: reversetext = '' if 'defocpair' in self.stackrunlogparams.keys(): defoctext = '--defocpair' else: defoctext = '' cmd = ''' makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s --no-invert %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2 ''' % (os.path.basename(newstackimagicfile), stackid, lowpasstext, highpasstext, partlimittext, reversetext, defoctext, normtext, unbinnedboxsize, bin, stackpathname, stackid, projectid, presetname, newstackrunname, newstackrundir, sessionname, sessionid) logfilepath = os.path.join(newstackrundir, 'frealignstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd, logfilepath) if returncode > 0: apDisplay.printError('Error in Frealign specific stack making') self.ImagicStackToFrealignMrcStack(newstackimagicfile) os.rename(newstackimagicfile[:-4] + '.mrc', newstackroot + '.mrc') self.setFrealignStack(newstackroot) # use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] #clean up rmfiles = glob.glob("*.box") for rmfile in rmfiles: apFile.removeFile(rmfile)
def getSessionDataFromReconId(reconid): stackid = apStack.getStackIdFromRecon(reconid) partdata = apStack.getOneParticleFromStackId(stackid, msg=False) sessiondata = partdata["particle"]["selectionrun"]["session"] return sessiondata
def start(self): self.addToLog('.... Setting up new ISAC job ....') self.addToLog('.... Making command for stack pre-processing ....') self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### send file to remotehost tasks = {} sfhed = self.stack['file'][:-4]+".hed" sfimg = self.stack['file'][:-4]+".img" tasks = self.addToTasks(tasks,"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s" % (sfhed,self.params['localhost'],self.params['remoterundir'],"start1.hed")) tasks = self.addToTasks(tasks,"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s" % (sfimg,self.params['localhost'],self.params['remoterundir'],"start1.img")) ### write Sparx jobfile: process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed") if os.path.isfile(self.params['localstack']): apFile.removeStack(self.params['localstack']) proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize)*self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) proccmd += " last="+str(self.params['numpart']-1) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) # apParam.runCmd(proccmd, "EMAN", verbose=True) self.addSimpleCommand('cd %s' % self.params['rundir']) self.addSimpleCommand(proccmd) sparxcmd = "sxcpy.py %s %s_1.hdf" % (self.params['localstack'], self.params['localstack'][:-4]) # apParam.runCmd(sparxcmd, "SPARX", verbose=True) self.addSimpleCommand(sparxcmd) self.addSimpleCommand("") ### write Sparx jobfile: run ISAC for i in range(self.params['generations']): sparxopts = " %s_%d.hdf" % (os.path.join(self.params['localstack'][:-4]), (i+1)) if self.params['ir'] is not None: sparxopts += " --ir %d" % int(float(apRecon.getComponentFromVector(self.params['ir'], i, splitter=":"))) if self.params['ou'] is not None: sparxopts += " --ou %d" % int(float(apRecon.getComponentFromVector(self.params['ou'], i, splitter=":"))) if self.params['rs'] is not None: sparxopts += " --rs %d" % int(float(apRecon.getComponentFromVector(self.params['rs'], i, splitter=":"))) if self.params['ts'] is not None: sparxopts += " --ts %.1f" % int(float(apRecon.getComponentFromVector(self.params['ts'], i, splitter=":"))) if self.params['xr'] is not None: sparxopts += " --xr %.1f" % int(float(apRecon.getComponentFromVector(self.params['xr'], i, splitter=":"))) if self.params['yr'] is not None: sparxopts += " --yr %.1f" % int(float(apRecon.getComponentFromVector(self.params['yr'], i, splitter=":"))) if self.params['maxit'] is not None: sparxopts += " --maxit %d" % int(float(apRecon.getComponentFromVector(self.params['maxit'], i, splitter=":"))) if self.params['FL'] is not None: sparxopts += " --FL %.2f" % int(float(apRecon.getComponentFromVector(self.params['FL'], i, splitter=":"))) if self.params['FH'] is not None: sparxopts += " --FH %.2f" % int(float(apRecon.getComponentFromVector(self.params['FH'], i, splitter=":"))) if self.params['FF'] is not None: sparxopts += " --FF %.2f" % int(float(apRecon.getComponentFromVector(self.params['FF'], i, splitter=":"))) if self.params['init_iter'] is not None: sparxopts += " --init_iter %d" % int(float(apRecon.getComponentFromVector(self.params['init_iter'], i, splitter=":"))) if self.params['main_iter'] is not None: sparxopts += " --main_iter %d" % int(float(apRecon.getComponentFromVector(self.params['main_iter'], i, splitter=":"))) if self.params['iter_reali'] is not None: sparxopts += " --iter_reali %d" % int(float(apRecon.getComponentFromVector(self.params['iter_reali'], i, splitter=":"))) if self.params['match_first'] is not None: sparxopts += " --match_first %d" % int(float(apRecon.getComponentFromVector(self.params['match_first'], i, splitter=":"))) if self.params['max_round'] is not None: sparxopts += " --max_round %d" % int(float(apRecon.getComponentFromVector(self.params['max_round'], i, splitter=":"))) if self.params['match_second'] is not None: sparxopts += " --match_second %d" % int(float(apRecon.getComponentFromVector(self.params['match_second'], i, splitter=":"))) if self.params['stab_ali'] is not None: sparxopts += " --stab_ali %d" % int(float(apRecon.getComponentFromVector(self.params['stab_ali'], i, splitter=":"))) if self.params['thld_err'] is not None: sparxopts += " --thld_err %.2f" % int(float(apRecon.getComponentFromVector(self.params['thld_err'], i, splitter=":"))) if self.params['indep_run'] is not None: sparxopts += " --indep_run %d" % int(float(apRecon.getComponentFromVector(self.params['indep_run'], i, splitter=":"))) if self.params['thld_grp'] is not None: sparxopts += " --thld_grp %d" % int(float(apRecon.getComponentFromVector(self.params['thld_grp'], i, splitter=":"))) if self.params['img_per_grp'] is not None: sparxopts += " --img_per_grp %d" % int(apRecon.getComponentFromVector(self.params['img_per_grp'], i)) sparxopts += " --generation %d" % (i+1) sparxexe = apParam.getExecPath("sxisac.py", die=True) mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+sparxexe+" "+sparxopts bn = os.path.basename(self.params['localstack'])[:-4] e2cmd = "e2proc2d.py %s_%d.hdf %s_%d.hdf --list=\"generation_%d_unaccounted.txt\"" % \ (bn, i+1, bn, i+2, i+1) self.addSimpleCommand(mpiruncmd) self.addSimpleCommand(e2cmd) # print self.tasks # print self.command_list # self.writeCommandListToFile() apParam.dumpParameters(self.params, "isac-"+self.params['timestamp']+"-params.pickle")
def start(self): self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) # symmetry info if self.params['sym'] == 'Icos': self.params['symtype'] = 'I' self.params['symfold'] = None else: self.params['symtype'] = self.params['sym'][0] self.params['symfold'] = int(self.params['sym'][1:]) # eman "d" symmetry is spider "ci" if self.params['symtype'].upper() == "D": self.params['symtype'] = "CI" # create symmetry doc file sydoc = "sym.spi" refine.symmetryDoc(self.params['symtype'], self.params['symfold'], sydoc) # convert incr to array of increments ang_inc = self.params['incr'].split(',') self.params['numiter'] = len(ang_inc) self.params['increments'] = [] for i in range(0, self.params['numiter']): self.params['increments'].append(int(ang_inc[i])) # convert stack to spider stack spiderstack = os.path.join(self.params['rundir'], 'start.spi') operations.stackToSpiderStack(self.stack['file'], spiderstack, apix=self.stack['apix'], boxsize=self.stack['boxsize'], numpart=self.params['numpart']) # create filtered stack spiderstackfilt = spiderstack if (self.params['lowpass'] + self.params['highpass']) > 0: spiderstackfilt = os.path.join(self.params['rundir'], 'start_filt.spi') operations.stackToSpiderStack(self.stack['file'], spiderstackfilt, apix=self.stack['apix'], boxsize=self.stack['boxsize'], lp=self.params['lowpass'], hp=self.params['highpass'], numpart=self.params['numpart']) # rescale initial model if necessary outvol = os.path.join(self.params['rundir'], "vol000.spi") apModel.rescaleModel(self.params['modelid'], outvol, self.stack['boxsize'], self.stack['apix'], spider=True) self.params['itervol'] = outvol for iter in range(1, self.params['numiter'] + 1): # create projections for projection matching apDisplay.printMsg("creating reference projections of volume: %s" % self.params['itervol']) projs, numprojs, ang, sel = refine.createProjections( incr=self.params['increments'][iter - 1], boxsz=self.stack['boxsize'], symfold=self.params['symfold'], invol=self.params['itervol'], rad=self.params['rad'], ) # run reference-based alignment apDisplay.printMsg("running reference-based alignment (AP MQ)") apmqfile = "apmq%03d.spi" % iter outang = "angular%03d.spi" % iter shf = "shifts%03d.spi" % iter shiftedStack = "parts_shifted.spi" apFile.removeFile(shf) refine.spiderAPMQ( projs=projs, numprojs=numprojs, tsearch=self.params['xysearch'], tstep=self.params['xystep'], firstRing=self.params['firstring'], lastRing=self.params['lastring'], stackfile=spiderstackfilt, nump=self.params['numpart'], ang=ang, apmqfile=apmqfile, outang=outang, nproc=self.params['proc'], ) # use cross-correlation to find the sub-pixel alignment # of the particles, # results will be saved in "peakfile.spi" apFile.removeFile(shiftedStack) # don't use MPI here - for some reason slower? mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False) apmqlist = refine.readDocFile(apmqfile) avgccrot = 0 apDisplay.printMsg("creating shifted stack") for p in range(0, self.params['numpart']): ref = int(float(apmqlist[p][2])) ccrot = float(apmqlist[p][3]) inplane = float(apmqlist[p][4]) avgccrot += ccrot # invert the sign - ref projs will be rotated inplane *= -1 # get corresponding projection if (ref <= 0): # mirror projection if necessary ref *= -1 refimg = spyder.fileFilter(projs) + "@" + str(ref) refine.mirrorImg(refimg, "_3", inMySpi=mySpi) img = "_3" else: img = spyder.fileFilter(projs) + "@" + str(ref) refine.rotAndShiftImg(img, "_2", inplane, inMySpi=mySpi) refine.maskImg("_2", "_3", self.params['rad'], "D", "E", center=int((self.stack['boxsize'] / 2) + 1), inMySpi=mySpi) # pad ref image & stack image to twice the size refine.padImg("_3", "_2", 2 * self.stack['boxsize'], "N", 1, 1, 0, inMySpi=mySpi) stackimg = spyder.fileFilter(spiderstack) + "@" + str(p + 1) refine.padImg(stackimg, "_1", 2 * self.stack['boxsize'], "B", 1, 1, inMySpi=mySpi) # calculate cross-correlation refine.getCC("_1", "_2", "_1", inMySpi=mySpi) # crop the correllation image to allowable shift amount shift = int(self.params['allowedShift'] * self.stack['boxsize']) dim = 2 * shift + 1 topleftx = self.stack['boxsize'] - shift + 1 refine.windowImg("_1", "_2", dim, topleftx, topleftx, inMySpi=mySpi) # find the sub-pixel location of cc peak mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17", "_2", "0") # create new stack of shifted particles shpos = spyder.fileFilter(shiftedStack) + "@" + str(p + 1) mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN") mySpi.toSpiderQuiet("GP x17", "_2", str(shift + 1) + "," + str(shift + 1)) refine.copyImg(stackimg, shpos, inMySpi=mySpi) mySpi.toSpiderQuiet("ELSE") #mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16") mySpi.toSpiderQuiet("SH F", stackimg, shpos, "-x15,-x16") mySpi.toSpiderQuiet("ENDIF") # save shifts to file mySpi.toSpiderQuiet("SD " + str(p + 1) + ",x15,x16,x17", spyder.fileFilter(shf)) mySpi.toSpiderQuiet("SD E", spyder.fileFilter(shf)) mySpi.close() # create class average images refine.createClassAverages( shiftedStack, projs, apmqfile, numprojs, self.stack['boxsize'], shifted=True, ) # rename class averages & variacnes for iteration cmd = "/bin/mv classes.hed classes.%d.hed;" % iter cmd += "/bin/mv classes.img classes.%d.img;" % iter cmd += "/bin/mv variances.hed variances.%d.hed;" % iter cmd += "/bin/mv variances.img variances.%d.img;" % iter proc = subprocess.Popen(cmd, shell=True) proc.wait() # calculate the stddev for the apmq cc's for throwing out particles avgccrot /= self.params['numpart'] stdccrot = 0 for p in range(0, self.params['numpart']): stdccrot += abs(float(apmqlist[p][3]) - avgccrot) stdccrot /= self.params['numpart'] cccutoff = avgccrot + (stdccrot * self.params['keepsig']) apDisplay.printMsg("average cc: %f" % avgccrot) apDisplay.printMsg("setting cutoff to: %f" % cccutoff) # create new selection file that only has particles with good cc's selectfile = "select%03d.spi" % iter apFile.removeFile(selectfile) mySpi = spyder.SpiderSession(nproc=self.params['proc'], dataext=".spi", logo=False, log=False) i = 1 for p in range(0, self.params['numpart']): ccrot = float(apmqlist[p][3]) if ccrot >= cccutoff: mySpi.toSpiderQuiet("x11=%d" % (p + 1)) mySpi.toSpiderQuiet("SD %d,x11" % i, spyder.fileFilter(selectfile)) i += 1 mySpi.close() # calculate the new 3d structure using centered projections # and the corrected angles from the angular doc file apDisplay.printMsg("creating 3d volume") out_rawvol = "vol_raw%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, selectfile, rad=self.params['rad'], ang=outang, out=out_rawvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['bpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc'], ) else: refine.backProjection(shiftedStack, selectfile, ang=outang, out=out_rawvol, sym=sydoc, nproc=self.params['proc']) # create even & odd select files apDisplay.printMsg("creating even/odd volumes") oddfile = "selectodd%03d.spi" % iter evenfile = "selecteven%03d.spi" % iter refine.docSplit(selectfile, oddfile, evenfile) # get the even & odd volumesa oddvol = "vol1%03d.spi" % iter evenvol = "vol2%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, oddfile, rad=self.params['rad'], ang=outang, out=oddvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc']) refine.iterativeBackProjection( shiftedStack, evenfile, rad=self.params['rad'], ang=outang, out=evenvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc']) else: refine.backProjection( shiftedStack, oddfile, ang=outang, out=oddvol, sym=sydoc, nproc=self.params['proc'], ) refine.backProjection( shiftedStack, evenfile, ang=outang, out=evenvol, sym=sydoc, nproc=self.params['proc'], ) # calculate the FSC apDisplay.printMsg("calculating FSC") fscfile = "fsc%03d.spi" % iter emanfsc = "fsc.eotest.%d" % iter refine.calcFSC(oddvol, evenvol, fscfile) # convert to eman-style fscfile refine.spiderFSCtoEMAN(fscfile, emanfsc) # calculate the resolution at 0.5 FSC & write to file res = apRecon.calcRes(emanfsc, self.stack['boxsize'], self.stack['apix']) restxt = "resolution.txt" if iter == 1 and os.path.isfile(restxt): os.remove(restxt) resfile = open(restxt, "a") resfile.write("iter %d:\t%.3f\n" % (iter, res)) resfile.close() # filter & normalize the volume to be used as a reference in the next round outvol = "vol%03d.spi" % iter emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % ( out_rawvol, outvol, self.stack['apix'], res, self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # create mrc files of volumes emancmd = "proc3d %s %s apix=%.3f mask=%d norm" % ( out_rawvol, "threed.%da.mrc" % iter, self.stack['apix'], self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % ( out_rawvol, "threed.%da.lp.mrc" % iter, self.stack['apix'], res, self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.param['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # set this model as start for next iteration, remove previous os.remove(self.params['itervol']) os.remove(out_rawvol) self.params['itervol'] = outvol # clean up directory apDisplay.printMsg("cleaning up directory") if os.path.isfile(oddvol): os.remove(oddvol) if os.path.isfile(evenvol): os.remove(evenvol) if os.path.isfile(ang): os.remove(ang) if os.path.isfile(sel): os.remove(sel) if os.path.isfile(projs): os.remove(projs) if os.path.isfile(oddfile): os.remove(oddfile) if os.path.isfile(evenfile): os.remove(evenfile) if os.path.isfile(emanfsc) and os.path.isfile(fscfile): os.remove(fscfile) if os.path.isfile(shiftedStack): os.remove(shiftedStack) os.remove(self.params['itervol'])
def start(self): # self.insertCL2DJob() self.stack = {} self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) if self.params['virtualdata'] is not None: self.stack['file'] = self.params['virtualdata']['filename'] else: self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) ### process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed") if os.path.isfile(self.params['localstack']): apFile.removeStack(self.params['localstack']) a = proc2dLib.RunProc2d() a.setValue('infile',self.stack['file']) a.setValue('outfile',self.params['localstack']) a.setValue('apix',self.stack['apix']) a.setValue('bin',self.params['bin']) a.setValue('last',self.params['numpart']-1) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: a.setValue('lowpass',self.params['lowpass']) if self.params['highpass'] is not None and self.params['highpass'] > 1: a.setValue('highpass',self.params['highpass']) if self.params['invert'] is True: a.setValue('invert',True) # clip not yet implemented # if self.params['clipsize'] is not None: # clipsize = int(self.clipsize)*self.params['bin'] # if clipsize % 2 == 1: # clipsize += 1 ### making sure that clipped boxsize is even # a.setValue('clip',clipsize) if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) #run proc2d a.run() if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']): apDisplay.printError("Missing particles in stack") ### setup Xmipp command aligntime = time.time() xmippopts = (" -i "+os.path.join(self.params['rundir'], self.params['localstack']) +" --nref "+str(self.params['numrefs']) +" --iter "+str(self.params['maxiter']) +" --odir "+str(self.params['rundir']) +" --oroot "+ "part"+str(self.params['timestamp']) +" --classifyAllImages" ) if self.params['correlation']: xmippopts += " --distance correlation" if self.params['classical']: xmippopts += " --classicalMultiref" ### use multi-processor command apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green") xmippexe = apParam.getExecPath(self.execFile, die=True) mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp 3", verbose=True, showcmd=True, logfile="xmipp.std") self.params['runtime'] = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime'])) ### post-processing # Create a stack for the class averages at each level Nlevels=glob.glob("level_*") for level in Nlevels: digits = level.split("_")[1] apParam.runCmd("xmipp_image_convert -i "+level+"/part"+self.params['timestamp']+"*xmd -o part" +self.params['timestamp']+"_level_"+digits+"_.hed", package="Xmipp 3", verbose=True) if self.params['align']: apParam.runCmd("xmipp_transform_geometry -i images.xmd -o %s_aligned.stk --apply_transform" % self.params['timestamp'], package="Xmipp 3", verbose=True) apParam.runCmd("xmipp_image_convert -i %s_aligned.xmd -o alignedStack.hed" % self.params['timestamp'], package="Xmipp 3", verbose=True) apFile.removeFile("%s_aligned.xmd" % self.params['timestamp']) apFile.removeFile("%s_aligned.stk" % self.params['timestamp']) self.parseOutput() apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle") ### upload results ... this used to be two separate operations, I'm combining into one self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle") self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin'] self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed")) ### create average of aligned stacks & insert aligned stack info lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1) apStack.averageStack(lastLevelStack) self.boxsize = apFile.getBoxSize(lastLevelStack)[0] self.insertCL2DParamsIntoDatabase() if self.runparams['align'] is True: self.insertAlignStackRunIntoDatabase("alignedStack.hed") self.calcResolution(self.Nlevels-1) self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1) ### loop over each class average stack & insert as clustering stacks self.insertClusterRunIntoDatabase() for level in range(self.Nlevels): ### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES if self.params['align'] is True: self.calcResolution(level) partdict = self.getClassificationAtLevel(level) for classnum in partdict: self.insertClusterStackIntoDatabase( "part"+self.params['timestamp']+"_level_%02d_.hed"%level, classnum+1, partdict[classnum], len(partdict)) self.clearIntermediateFiles()
def start(self): self.runtime = 0 self.partlist = [] self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.clipsize = int(math.floor(self.stack['boxsize']/self.params['bin']/2.0)*self.params['bin']*2.0) #self.checkRunNamePath() ### convert stack to spider spiderstack = self.createSpiderFile() #create template stack templatestack = self.createTemplateStack() #create orientation reference orientref = self.createOrientationReference() ################################################################### aligntime = time.time() ### create batch file batchfilepath = self.setupBatchFile(spiderstack, templatestack, orientref) ### run the spider alignment apDisplay.printColor("Running iterative ref-classification and free-alignment with spider","cyan") self.runSpiderBatch( batchfilepath, spiderstack ) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ################################################################### ### remove unaligned spider stack apDisplay.printMsg("Removing un-aligned stack: "+spiderstack) apFile.removeFile(spiderstack, warn=False) ### check to be sure files exist avgfile = os.path.join(self.params['rundir'], "alignstack.spi") #class averages if not os.path.isfile(avgfile): apDisplay.printError("Final stack of aligned particles does not exist.") ### convert stacks to imagic self.convertStack2Imagic("alignstack.spi") self.convertStack2Imagic("avg.spi") ### make alignment average in mrc format emancmd = "proc2d avg.spi average.mrc average" apEMAN.executeEmanCmd(emancmd) inserttime = time.time() if self.params['commit'] is True: apDisplay.printWarning("committing results to DB") self.runtime = aligntime self.insertEdIterRun(insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def start(self): self.runtime = 0 self.partlist = [] self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.clipsize = int( math.floor(self.stack['boxsize'] / self.params['bin'] / 2.0) * self.params['bin'] * 2.0) #self.checkRunNamePath() ### convert stack to spider spiderstack = self.createSpiderFile() #create template stack templatestack = self.createTemplateStack() #create orientation reference orientref = self.createOrientationReference() ################################################################### aligntime = time.time() ### create batch file batchfilepath = self.setupBatchFile(spiderstack, templatestack, orientref) ### run the spider alignment apDisplay.printColor( "Running iterative ref-classification and free-alignment with spider", "cyan") self.runSpiderBatch(batchfilepath, spiderstack) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ################################################################### ### remove unaligned spider stack apDisplay.printMsg("Removing un-aligned stack: " + spiderstack) apFile.removeFile(spiderstack, warn=False) ### check to be sure files exist avgfile = os.path.join(self.params['rundir'], "alignstack.spi") #class averages if not os.path.isfile(avgfile): apDisplay.printError( "Final stack of aligned particles does not exist.") ### convert stacks to imagic self.convertStack2Imagic("alignstack.spi") self.convertStack2Imagic("avg.spi") ### make alignment average in mrc format emancmd = "proc2d avg.spi average.mrc average" apEMAN.executeEmanCmd(emancmd) inserttime = time.time() if self.params['commit'] is True: apDisplay.printWarning("committing results to DB") self.runtime = aligntime self.insertEdIterRun(insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) apDisplay.printMsg("Database Insertion time: " + apDisplay.timeString(inserttime))
def start(self): # self.insertCL2DJob() self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join( self.params['rundir'], self.params['timestamp'] + ".hed") if os.path.isfile(self.params['localstack']): apFile.removeStack(self.params['localstack']) proccmd = "proc2d " + self.stack['file'] + " " + self.params[ 'localstack'] + " apix=" + str(self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], clipsize, clipsize) proccmd += " last=" + str(self.params['numpart'] - 1) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp=" + str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp=" + str(self.params['lowpass']) apParam.runCmd(proccmd, "EMAN", verbose=True) if self.params['numpart'] != apFile.numImagesInStack( self.params['localstack']): apDisplay.printError("Missing particles in stack") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles( self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params['rundir'], self.partlistdocfile) + " -codes " + str(self.params['numrefs']) + " -iter " + str(self.params['maxiter']) + " -o " + os.path.join( self.params['rundir'], "part" + self.params['timestamp'])) if self.params['fast']: xmippopts += " -fast " if self.params['correlation']: xmippopts += " -useCorrelation " if self.params['classical']: xmippopts += " -classicalMultiref " if self.params['align']: xmippopts += " -alignImages " ### use multi-processor command apDisplay.printColor( "Using " + str(self.params['nproc']) + " processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True) mpiruncmd = self.mpirun + " -np " + str( self.params['nproc']) + " " + xmippexe + " " + xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std") self.params['runtime'] = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(self.params['runtime'])) ### minor post-processing self.createReferenceStack() self.parseOutput() self.clearIntermediateFiles() # self.readyUploadFlag() apParam.dumpParameters( self.params, "cl2d-" + self.params['timestamp'] + "-params.pickle") ### upload results ... this used to be two separate operations, I'm combining into one self.runparams = apParam.readRunParameters("cl2d-" + self.params['timestamp'] + "-params.pickle") self.apix = apStack.getStackPixelSizeFromStackId( self.runparams['stackid']) * self.runparams['bin'] self.Nlevels = len( glob.glob("part" + self.params['timestamp'] + "_level_??_.hed")) ### create average of aligned stacks & insert aligned stack info lastLevelStack = "part" + self.params[ 'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1) apStack.averageStack(lastLevelStack) self.boxsize = apFile.getBoxSize(lastLevelStack)[0] self.insertCL2DParamsIntoDatabase() if self.runparams['align'] is True: self.insertAlignStackRunIntoDatabase("alignedStack.hed") self.calcResolution(self.Nlevels - 1) self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1) ### loop over each class average stack & insert as clustering stacks self.insertClusterRunIntoDatabase() for level in range(self.Nlevels): ### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES if self.params['align'] is True: self.calcResolution(level) partdict = self.getClassificationAtLevel(level) for classnum in partdict: self.insertClusterStackIntoDatabase( "part" + self.params['timestamp'] + "_level_%02d_.hed" % level, classnum + 1, partdict[classnum], len(partdict))
def start(self): self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### test insert to make sure data is not overwritten self.params['runtime'] = 0 #self.checkDuplicateRefBasedRun() ### set box size self.boxsize = int(math.floor(self.stack['boxsize']/self.params['bin']/2.0))*2 ### convert stack to spider spiderstack = self.createSpiderFile() ### create template stack templatestack = self.createTemplateStack() ### run the alignment aligntime = time.time() usestack = spiderstack oldpartlist = None for i in range(self.params['numiter']): iternum = i+1 apDisplay.printColor("\n\nITERATION "+str(iternum), "green") alignedstack, partlist = self.runAlignmentGPU( usestack, templatestack, spiderstack, self.params['xysearch'], self.params['xystep'], self.params['firstring'], self.params['lastring'], iternum=iternum, oldpartlist=oldpartlist) oldpartlist = partlist usestack = alignedstack templatestack = self.updateTemplateStack(alignedstack, partlist, iternum) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ### remove large, worthless stack spiderstack = os.path.join(self.params['rundir'], "start.spi") apDisplay.printMsg("Removing un-aligned stack: "+spiderstack) apFile.removeFile(spiderstack, warn=True) ### convert aligned stack to imagic finalspistack = "aligned.spi" shutil.move(alignedstack, finalspistack) imagicstack = "aligned.hed" apFile.removeStack(imagicstack) emancmd = "proc2d "+finalspistack+" "+imagicstack apEMAN.executeEmanCmd(emancmd, verbose=True) ### average stack apStack.averageStack(imagicstack) ### calculate resolution for each reference apix = self.stack['apix']*self.params['bin'] self.calcResolution(partlist, imagicstack, apix) if self.params['commit'] is True: apDisplay.printMsg("committing results to DB") self.params['runtime'] = aligntime self.insertRefBasedRun(partlist, imagicstack, insert=True) else: apDisplay.printWarning("not committing results to DB") ### remove temporary files apFile.removeFilePattern("alignments/alignedstack*.spi") apFile.removeFile(finalspistack)
def convertToRefineStack(self): ''' The stack is remaked without ctf correction and without invertion (ccd) ''' newstackroot = os.path.join(self.params['rundir'],os.path.basename(self.stack['file'])[:-4]) if self.params['paramonly'] is True: print 'newstackroot',newstackroot self.setFrealignStack(newstackroot) return # copy existing refined stack if possible existing_refine_stack = apStack.getExistingRefineStack(self.stack['data'],'frealign',False,self.params['last'],self.params['bin'],lowpass=self.params['lowpass'],highpass=self.params['highpass']) if existing_refine_stack: if existing_refine_stack != os.path.join(self.params['rundir'],newstackroot+'.mrc'): shutil.copyfile(existing_refine_stack,newstackroot+'.mrc') new_partorderfile = os.path.join(self.params['rundir'],'stackpartorder.list') existing_partorderfile = os.path.join(os.path.dirname(existing_refine_stack),'stackpartorder.list') # particle order list also need to be copied if not os.path.isfile(new_partorderfile) and os.path.isfile(existing_partorderfile): shutil.copyfile(existing_partorderfile,new_partorderfile) self.setFrealignStack(newstackroot) return if self.no_ctf_correction: self.ImagicStackToFrealignMrcStack(self.stack['file']) self.setFrealignStack(newstackroot) return # stack need to be remade without ctf correction apDisplay.printWarning('Frealign needs a stack without ctf correction. A new stack is being made....') stackdata = self.stack['data'] madestackdata = self.getOriginalStackMadeByMakeStack() stackid = stackdata.dbid stackruns = apStack.getStackRunsFromStack(madestackdata) # To Do: need to handle combined stack stackrun = stackruns[0] stackpathname = os.path.basename(stackdata['path']['path']) totalpart = apStack.getNumberStackParticlesFromId(stackid) if not self.params['last']: numpart = totalpart else: numpart = min(self.params['last'],totalpart) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir,'temp.hed') # use first particle image to get presetname oneparticle = apStack.getOneParticleFromStackId(stackid, particlenumber=1) preset =oneparticle['particle']['image']['preset'] if preset: presetname = preset['name'] else: presetname = 'manual' # use first stack run to get parameters paramdata = stackrun['stackParams'] # binning is combination of the original binning of the stack and the preparation binnning bin = paramdata['bin']*self.params['bin'] unbinnedboxsize = self.stack['boxsize']*paramdata['bin'] lowpasstext = self.setArgText('lowpass',(self.params['lowpass'],paramdata['lowpass']),False) highpasstext = self.setArgText('highpass',(self.params['highpass'],paramdata['highpass']),True) partlimittext = self.setArgText('partlimit',(numpart,),False) xmipp_normtext = self.setArgText('xmipp-normalize',(paramdata['xmipp-norm'],),True) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] # The assumption is that the image is from ice grid and digital camera (black particles on white background if 'reverse' in self.stackrunlogparams.keys(): reversetext = '--reverse' else: reversetext = '' if 'defocpair' in self.stackrunlogparams.keys(): defoctext = '--defocpair' else: defoctext = '' cmd = ''' makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s --no-invert --normalized %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2 ''' % (os.path.basename(newstackimagicfile),stackid,lowpasstext,highpasstext,partlimittext,reversetext,defoctext,xmipp_normtext,unbinnedboxsize,bin,stackpathname,stackid,projectid,presetname,newstackrunname,newstackrundir,sessionname,sessionid) logfilepath = os.path.join(newstackrundir,'frealignstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd,logfilepath) if returncode > 0: apDisplay.printError('Error in Frealign specific stack making') self.ImagicStackToFrealignMrcStack(newstackimagicfile) os.rename(newstackimagicfile[:-4]+'.mrc',newstackroot+'.mrc') self.setFrealignStack(newstackroot) # use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] #clean up rmfiles = glob.glob("*.box") for rmfile in rmfiles: apFile.removeFile(rmfile)
def start(self): self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) # symmetry info if self.params['sym']=='Icos': self.params['symtype']='I' self.params['symfold']=None else: self.params['symtype']=self.params['sym'][0] self.params['symfold']=int(self.params['sym'][1:]) # eman "d" symmetry is spider "ci" if self.params['symtype'].upper()=="D": self.params['symtype'] = "CI" # create symmetry doc file sydoc="sym.spi" refine.symmetryDoc(self.params['symtype'],self.params['symfold'],sydoc) # convert incr to array of increments ang_inc=self.params['incr'].split(',') self.params['numiter'] = len(ang_inc) self.params['increments']=[] for i in range(0,self.params['numiter']): self.params['increments'].append(int(ang_inc[i])) # convert stack to spider stack spiderstack=os.path.join(self.params['rundir'],'start.spi') operations.stackToSpiderStack( self.stack['file'], spiderstack, apix=self.stack['apix'], boxsize=self.stack['boxsize'], numpart=self.params['numpart'] ) # create filtered stack spiderstackfilt=spiderstack if (self.params['lowpass']+self.params['highpass']) > 0: spiderstackfilt=os.path.join(self.params['rundir'],'start_filt.spi') operations.stackToSpiderStack( self.stack['file'], spiderstackfilt, apix=self.stack['apix'], boxsize=self.stack['boxsize'], lp=self.params['lowpass'], hp=self.params['highpass'], numpart=self.params['numpart'] ) # rescale initial model if necessary outvol = os.path.join(self.params['rundir'],"vol000.spi") apModel.rescaleModel(self.params['modelid'], outvol, self.stack['boxsize'], self.stack['apix'], spider=True) self.params['itervol']=outvol for iter in range(1,self.params['numiter']+1): # create projections for projection matching apDisplay.printMsg("creating reference projections of volume: %s" % self.params['itervol']) projs,numprojs,ang,sel = refine.createProjections( incr=self.params['increments'][iter-1], boxsz=self.stack['boxsize'], symfold=self.params['symfold'], invol=self.params['itervol'], rad=self.params['rad'], ) # run reference-based alignment apDisplay.printMsg("running reference-based alignment (AP MQ)") apmqfile = "apmq%03d.spi" % iter outang = "angular%03d.spi" % iter shf = "shifts%03d.spi" % iter shiftedStack="parts_shifted.spi" apFile.removeFile(shf) refine.spiderAPMQ( projs=projs, numprojs=numprojs, tsearch=self.params['xysearch'], tstep=self.params['xystep'], firstRing=self.params['firstring'], lastRing=self.params['lastring'], stackfile=spiderstackfilt, nump=self.params['numpart'], ang=ang, apmqfile=apmqfile, outang=outang, nproc=self.params['proc'], ) # use cross-correlation to find the sub-pixel alignment # of the particles, # results will be saved in "peakfile.spi" apFile.removeFile(shiftedStack) # don't use MPI here - for some reason slower? mySpi=spyder.SpiderSession(dataext=".spi", logo=False, log=False) apmqlist = refine.readDocFile(apmqfile) avgccrot = 0 apDisplay.printMsg("creating shifted stack") for p in range(0,self.params['numpart']): ref=int(float(apmqlist[p][2])) ccrot=float(apmqlist[p][3]) inplane=float(apmqlist[p][4]) avgccrot+=ccrot # invert the sign - ref projs will be rotated inplane*=-1 # get corresponding projection if (ref <= 0): # mirror projection if necessary ref*=-1 refimg=spyder.fileFilter(projs)+"@"+str(ref) refine.mirrorImg(refimg,"_3",inMySpi=mySpi) img="_3" else: img=spyder.fileFilter(projs)+"@"+str(ref) refine.rotAndShiftImg(img,"_2",inplane,inMySpi=mySpi) refine.maskImg("_2","_3",self.params['rad'],"D","E", center=int((self.stack['boxsize']/2)+1), inMySpi=mySpi) # pad ref image & stack image to twice the size refine.padImg("_3","_2",2*self.stack['boxsize'],"N",1,1,0,inMySpi=mySpi) stackimg=spyder.fileFilter(spiderstack)+"@"+str(p+1) refine.padImg(stackimg,"_1",2*self.stack['boxsize'],"B",1,1,inMySpi=mySpi) # calculate cross-correlation refine.getCC("_1","_2","_1",inMySpi=mySpi) # crop the correllation image to allowable shift amount shift=int(self.params['allowedShift']*self.stack['boxsize']) dim=2*shift+1 topleftx=self.stack['boxsize']-shift+1 refine.windowImg("_1","_2",dim,topleftx,topleftx,inMySpi=mySpi) # find the sub-pixel location of cc peak mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17","_2","0") # create new stack of shifted particles shpos=spyder.fileFilter(shiftedStack)+"@"+str(p+1) mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN") mySpi.toSpiderQuiet("GP x17","_2",str(shift+1)+","+str(shift+1)) refine.copyImg(stackimg,shpos,inMySpi=mySpi) mySpi.toSpiderQuiet("ELSE") #mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16") mySpi.toSpiderQuiet("SH F",stackimg,shpos,"-x15,-x16") mySpi.toSpiderQuiet("ENDIF") # save shifts to file mySpi.toSpiderQuiet("SD "+str(p+1)+",x15,x16,x17",spyder.fileFilter(shf)) mySpi.toSpiderQuiet("SD E",spyder.fileFilter(shf)) mySpi.close() # create class average images refine.createClassAverages( shiftedStack, projs, apmqfile, numprojs, self.stack['boxsize'], shifted=True, ) # rename class averages & variacnes for iteration cmd="/bin/mv classes.hed classes.%d.hed;" % iter cmd+="/bin/mv classes.img classes.%d.img;" % iter cmd+="/bin/mv variances.hed variances.%d.hed;" % iter cmd+="/bin/mv variances.img variances.%d.img;" % iter proc = subprocess.Popen(cmd, shell=True) proc.wait() # calculate the stddev for the apmq cc's for throwing out particles avgccrot/=self.params['numpart'] stdccrot = 0 for p in range(0,self.params['numpart']): stdccrot+=abs(float(apmqlist[p][3])-avgccrot) stdccrot/=self.params['numpart'] cccutoff=avgccrot+(stdccrot*self.params['keepsig']) apDisplay.printMsg("average cc: %f" %avgccrot) apDisplay.printMsg("setting cutoff to: %f" %cccutoff) # create new selection file that only has particles with good cc's selectfile="select%03d.spi" % iter apFile.removeFile(selectfile) mySpi = spyder.SpiderSession(nproc=self.params['proc'],dataext=".spi", logo=False, log=False) i=1 for p in range(0,self.params['numpart']): ccrot=float(apmqlist[p][3]) if ccrot>=cccutoff: mySpi.toSpiderQuiet("x11=%d" % (p+1)) mySpi.toSpiderQuiet("SD %d,x11" % i,spyder.fileFilter(selectfile)) i+=1 mySpi.close() # calculate the new 3d structure using centered projections # and the corrected angles from the angular doc file apDisplay.printMsg("creating 3d volume") out_rawvol="vol_raw%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, selectfile, rad=self.params['rad'], ang=outang, out=out_rawvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['bpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc'], ) else: refine.backProjection( shiftedStack, selectfile, ang=outang, out=out_rawvol, sym=sydoc, nproc=self.params['proc'] ) # create even & odd select files apDisplay.printMsg("creating even/odd volumes") oddfile="selectodd%03d.spi" % iter evenfile="selecteven%03d.spi" % iter refine.docSplit(selectfile,oddfile,evenfile) # get the even & odd volumesa oddvol="vol1%03d.spi" % iter evenvol="vol2%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, oddfile, rad=self.params['rad'], ang=outang, out=oddvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc'] ) refine.iterativeBackProjection( shiftedStack, evenfile, rad=self.params['rad'], ang=outang, out=evenvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc'] ) else: refine.backProjection( shiftedStack, oddfile, ang=outang, out=oddvol, sym=sydoc, nproc=self.params['proc'], ) refine.backProjection( shiftedStack, evenfile, ang=outang, out=evenvol, sym=sydoc, nproc=self.params['proc'], ) # calculate the FSC apDisplay.printMsg("calculating FSC") fscfile="fsc%03d.spi" % iter emanfsc="fsc.eotest.%d" % iter refine.calcFSC(oddvol,evenvol,fscfile) # convert to eman-style fscfile refine.spiderFSCtoEMAN(fscfile,emanfsc) # calculate the resolution at 0.5 FSC & write to file res=apRecon.calcRes(emanfsc,self.stack['boxsize'],self.stack['apix']) restxt="resolution.txt" if iter==1 and os.path.isfile(restxt): os.remove(restxt) resfile=open(restxt,"a") resfile.write("iter %d:\t%.3f\n" % (iter,res)) resfile.close() # filter & normalize the volume to be used as a reference in the next round outvol="vol%03d.spi" % iter emancmd="proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % (out_rawvol,outvol,self.stack['apix'],res,self.params['rad']) if self.params['imask'] is not None: emancmd+=" imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # create mrc files of volumes emancmd="proc3d %s %s apix=%.3f mask=%d norm" % (out_rawvol,"threed.%da.mrc" % iter, self.stack['apix'],self.params['rad']) if self.params['imask'] is not None: emancmd+=" imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) emancmd="proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % (out_rawvol,"threed.%da.lp.mrc" % iter, self.stack['apix'],res, self.params['rad']) if self.params['imask'] is not None: emancmd+=" imask=%d" % self.param['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # set this model as start for next iteration, remove previous os.remove(self.params['itervol']) os.remove(out_rawvol) self.params['itervol']=outvol # clean up directory apDisplay.printMsg("cleaning up directory") if os.path.isfile(oddvol): os.remove(oddvol) if os.path.isfile(evenvol): os.remove(evenvol) if os.path.isfile(ang): os.remove(ang) if os.path.isfile(sel): os.remove(sel) if os.path.isfile(projs): os.remove(projs) if os.path.isfile(oddfile): os.remove(oddfile) if os.path.isfile(evenfile): os.remove(evenfile) if os.path.isfile(emanfsc) and os.path.isfile(fscfile): os.remove(fscfile) if os.path.isfile(shiftedStack): os.remove(shiftedStack) os.remove(self.params['itervol'])
def start(self): # self.insertCL2DJob() self.stack = {} self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) if self.params['virtualdata'] is not None: self.stack['file'] = self.params['virtualdata']['filename'] else: self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) ### process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed") if os.path.isfile(self.params['localstack']): apFile.removeStack(self.params['localstack']) a = proc2dLib.RunProc2d() a.setValue('infile',self.stack['file']) a.setValue('outfile',self.params['localstack']) a.setValue('apix',self.stack['apix']) a.setValue('bin',self.params['bin']) a.setValue('last',self.params['numpart']-1) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: a.setValue('lowpass',self.params['lowpass']) if self.params['highpass'] is not None and self.params['highpass'] > 1: a.setValue('highpass',self.params['highpass']) if self.params['invert'] is True: a.setValue('invert',True) # clip not yet implemented # if self.params['clipsize'] is not None: # clipsize = int(self.clipsize)*self.params['bin'] # if clipsize % 2 == 1: # clipsize += 1 ### making sure that clipped boxsize is even # a.setValue('clip',clipsize) if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) #run proc2d a.run() if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']): apDisplay.printError("Missing particles in stack") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " +" -i "+os.path.join(self.params['rundir'], self.partlistdocfile) +" -codes "+str(self.params['numrefs']) +" -iter "+str(self.params['maxiter']) +" -o "+os.path.join(self.params['rundir'], "part"+self.params['timestamp']) ) if self.params['fast']: xmippopts += " -fast " if self.params['correlation']: xmippopts += " -useCorrelation " if self.params['classical']: xmippopts += " -classicalMultiref " if self.params['align']: xmippopts += " -alignImages " ### use multi-processor command apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True) mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std") self.params['runtime'] = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime'])) ### minor post-processing self.createReferenceStack() self.parseOutput() self.clearIntermediateFiles() # self.readyUploadFlag() apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle") ### upload results ... this used to be two separate operations, I'm combining into one self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle") self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin'] self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed")) ### create average of aligned stacks & insert aligned stack info lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1) apStack.averageStack(lastLevelStack) self.boxsize = apFile.getBoxSize(lastLevelStack)[0] self.insertCL2DParamsIntoDatabase() if self.runparams['align'] is True: self.insertAlignStackRunIntoDatabase("alignedStack.hed") self.calcResolution(self.Nlevels-1) self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1) ### loop over each class average stack & insert as clustering stacks self.insertClusterRunIntoDatabase() for level in range(self.Nlevels): ### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES if self.params['align'] is True: self.calcResolution(level) partdict = self.getClassificationAtLevel(level) for classnum in partdict: self.insertClusterStackIntoDatabase( "part"+self.params['timestamp']+"_level_%02d_.hed"%level, classnum+1, partdict[classnum], len(partdict))
def checkConflicts(self): ### first get all stack data self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### modify boxsize and pixelsize according to binning self.boxsize = self.stack['boxsize'] self.clipsize = int( math.floor(self.boxsize / float(self.params['bin'] * 2))) * 2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" % (self.params['clipsize'], self.clipsize)) self.clipsize = self.params['clipsize'] self.apix = self.stack['apix'] if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even self.boxsize = clipsize self.apix = self.apix * self.params['bin'] ### basic error checking if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['ncls'] is None: apDisplay.printError("a number of classes was not provided") maxparticles = 1000000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile))) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) if self.params['numpart'] > 5000: apDisplay.printWarning( "initial model calculation may not work with less than 5000 particles" ) self.mpirun = self.checkMPI() # if self.mpirun is None: # apDisplay.printError("There is no MPI installed") if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() ### SIMPLE defaults and error checking if self.params['ring2'] is None: self.params['ring2'] = (self.boxsize / 2) - 2 if self.params['ncls'] > 2000: apDisplay.printError( "number of classes should be less than 2000 for subsequent ORIGAMI run to work" ) if self.params['ncls'] > self.params['numpart']: self.params['ncls'] = self.params['numpart'] / self.params['minp'] if self.params['mask'] is None: self.params['mask'] = (self.boxsize / 2) - 2 if self.params['mw'] is None: apDisplay.printError( "please specify the molecular weight (in kDa)")
def start(self): self.insertMaxLikeJob() self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.estimateIterTime() self.dumpParameters() ### process stack to local temp file proccmd = "proc2d "+self.stack['file']+" temp.hed apix="+str(self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize)*self.params['bin'] proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) proccmd += " last="+str(self.params['numpart']-1) apEMAN.executeEmanCmd(proccmd, verbose=True) ### process stack to final file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") proccmd = "proc2d temp.hed "+self.params['localstack']+" apix="+str(self.stack['apix']*self.params['bin']) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) apEMAN.executeEmanCmd(proccmd, verbose=True) apFile.removeStack("temp.hed") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) ### convert stack into single spider files templateselfile = self.initializeTemplates() ### setup Xmipp command aligntime = time.time() xmippopts = ( " " +" -i "+os.path.join(self.params['rundir'], self.partlistdocfile) +" -iter "+str(self.params['maxiter']) +" -ref "+templateselfile +" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp) +" -psi_step "+str(self.params['psistep']) ) ### fast mode if self.params['fast'] is True: xmippopts += " -fast " if self.params['fastmode'] == "narrow": xmippopts += " -C 1e-10 " elif self.params['fastmode'] == "wide": xmippopts += " -C 1e-18 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " ### normalization if self.params['norm'] is True: xmippopts += " -norm " ### find number of processors if self.params['nproc'] is None: nproc = nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] mpirun = self.checkMPI() if nproc > 2 and mpirun is not None: ### use multi-processor apDisplay.printColor("Using "+str(nproc-1)+" processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True) mpiruncmd = mpirun+" -np "+str(nproc-1)+" "+xmippexe+" "+xmippopts self.writeXmippLog(mpiruncmd) apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True) else: ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe+" "+xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ### minor post-processing self.createReferenceStack() self.readyUploadFlag() self.dumpParameters()