def makeNewStack(oldstack, newstack, listfile=None, remove=False, bad=False):
"""
selects particular particles from a stack
"""
if not os.path.isfile(oldstack):
apDisplay.printWarning("could not find old stack: " + oldstack)
if os.path.isfile(newstack):
if remove is True:
apDisplay.printWarning("removing old stack: " + newstack)
time.sleep(2)
apFile.removeStack(newstack)
else:
apDisplay.printError("new stack already exists: " + newstack)
apDisplay.printMsg("creating a new stack\n\t" + newstack +
"\nfrom the oldstack\n\t" + oldstack + "\n")
a = proc2dLib.RunProc2d()
a.setValue('infile', oldstack)
a.setValue('outfile', newstack)
if listfile is not None:
a.setValue('list', listfile)
a.run()
if bad is True and listfile is not None:
### run only if num bad particles < num good particles
newstacknumpart = apFile.numImagesInStack(newstack)
oldstacknumpart = apFile.numImagesInStack(oldstack)
if newstacknumpart > oldstacknumpart / 2:
### create bad.hed stack with all bad particles
badstack = os.path.join(os.path.dirname(newstack), "bad.hed")
emancmd = "proc2d %s %s exclude=%s" % (oldstack, badstack,
listfile)
apEMAN.executeEmanCmd(emancmd, verbose=True)
else:
apDisplay.printMsg(
"Rejecting more particles than keeping, not creating a bad stack"
)
return
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local file
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
a.setValue('append',False)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert') is True
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
# clip not yet implemented
# self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### stack is needed by uploadMaxlike.py
#apFile.removeStack(self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -nref "+str(self.params['numrefs'])
+" -iter "+str(self.params['maxiter'])
+" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp)
+" -psi_step "+str(self.params['psistep'])
)
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-17 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### use student's T distribution
if self.params['student'] is True:
xmippopts += " -student "
### write cluster job file
if self.params['cluster'] is True:
self.writeClusterJobFile()
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
self.estimateIterTime()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using "+str(nproc)+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun+" -np "+str(nproc)+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe+" "+xmippopts
self.writeXmippLog(xmippcmd)
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
stackdata = apStack.getOnlyStackData(params['stackid'])
stackpath = stackdata['path']['path']
# generate stack if it doesn't exist.
if not os.path.isdir(stackpath):
os.makedirs(stackpath)
fname = os.path.join(stackpath, stackdata['name'])
# check if stack file already exists
if os.path.isfile(fname):
apDisplay.printError("file: '%s' already exists" % fname)
vstackdata = apStack.getVirtualStackParticlesFromId(params['stackid'])
plist = [int(p['particleNumber']) - 1 for p in vstackdata['particles']]
a = proc2dLib.RunProc2d()
a.setValue('infile', vstackdata['filename'])
a.setValue('outfile', fname)
a.setValue('list', plist)
a.setValue('apix', apStack.getStackPixelSizeFromStackId(params['stackid']))
apDisplay.printMsg("generating stack: '%s' with %i particles" %
(fname, len(plist)))
a.run()
outavg = os.path.join(stackpath, "average.mrc")
if not os.path.isfile(outavg):
apStack.averageStack(stack=fname, outfile=outavg)
montageimg = os.path.join(stackpath, "montage%i.png" % params['stackid'])
if not os.path.isfile(montageimg):
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### setup Xmipp command
aligntime = time.time()
xmippopts = (" -i "+os.path.join(self.params['rundir'], self.params['localstack'])
+" --nref "+str(self.params['numrefs'])
+" --iter "+str(self.params['maxiter'])
+" --odir "+str(self.params['rundir'])
+" --oroot "+ "part"+str(self.params['timestamp'])
+" --classifyAllImages"
)
if self.params['correlation']:
xmippopts += " --distance correlation"
if self.params['classical']:
xmippopts += " --classicalMultiref"
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath(self.execFile, die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp 3", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### post-processing
# Create a stack for the class averages at each level
Nlevels=glob.glob("level_*")
for level in Nlevels:
digits = level.split("_")[1]
apParam.runCmd("xmipp_image_convert -i "+level+"/part"+self.params['timestamp']+"*xmd -o part"
+self.params['timestamp']+"_level_"+digits+"_.hed", package="Xmipp 3", verbose=True)
if self.params['align']:
apParam.runCmd("xmipp_transform_geometry -i images.xmd -o %s_aligned.stk --apply_transform" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apParam.runCmd("xmipp_image_convert -i %s_aligned.xmd -o alignedStack.hed" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apFile.removeFile("%s_aligned.xmd" % self.params['timestamp'])
apFile.removeFile("%s_aligned.stk" % self.params['timestamp'])
self.parseOutput()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
self.clearIntermediateFiles()
def start(self):
self.insertTopolRepJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
self.dumpParameters()
self.params['canexe'] = self.getCANPath()
### process stack to local file
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
a.setValue('append',False)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
if self.params['premask'] is True and self.params['mramethod'] != 'imagic':
a.setValue('mask',self.params['mask'])
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
if self.params['uploadonly'] is not True:
if os.path.isfile(os.path.join(self.params['rundir'],"stack.hed")):
self.params['localstack']=os.path.join(self.params['rundir'],"stack.hed")
else:
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### IMAGIC mask particles before alignment
if self.params['premask'] is True and self.params['mramethod'] == 'imagic':
# convert mask to fraction for imagic
maskfrac = self.workingmask*2/self.workingboxsize
maskstack = imagicFilters.softMask(self.params['localstack'],mask=maskfrac)
shutil.move(maskstack+".hed",os.path.splitext(self.params['localstack'])[0]+".hed")
shutil.move(maskstack+".img",os.path.splitext(self.params['localstack'])[0]+".img")
origstack = self.params['localstack']
### find number of processors
# if self.params['nproc'] is None:
self.params['nproc'] = apParam.getNumProcessors()
if self.params['uploadonly'] is not True:
aligntime = time.time()
# run through iterations
for i in range(0,self.params['iter']+1):
# move back to starting directory
os.chdir(self.params['rundir'])
# set up next iteration directory
self.params['currentiter'] = i
self.params['iterdir'] = os.path.abspath("iter%02i" % i)
self.params['iterdir'] = string.replace(self.params['iterdir'],"/jetstor/APPION","")
if os.path.exists(self.params['iterdir']):
apDisplay.printError("Error: directory '%s' exists, aborting alignment" % self.params['iterdir'])
# create directory for iteration
os.makedirs(self.params['iterdir'])
os.chdir(self.params['iterdir'])
# if at first iteration, create initial class averages
if i == 0:
# first rewrite localstack headers if particles not pre-masked
if self.params['premask'] is False and self.params['mramethod'] == "imagic":
imagicFilters.takeoverHeaders(self.params['localstack'],self.params['numpart'],self.workingboxsize)
self.params['alignedstack'] = os.path.splitext(self.params['localstack'])[0]
if self.params['msamethod']=='imagic':
self.runIMAGICmsa()
else:
self.runCAN()
continue
# using references from last iteration, run multi-ref alignment
if self.params['mramethod'] == "imagic":
# rewrite class headers
imagicFilters.takeoverHeaders(self.params['currentcls'],self.params['currentnumclasses'],self.workingboxsize)
self.runIMAGICmra()
else:
self.runEMANmra()
# create class averages from aligned stack
if self.params['msamethod']=='imagic':
self.runIMAGICmsa()
else:
self.runCAN()
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
## set upload information params:
else:
## get last iteration
alliters = glob.glob("iter*")
alliters.sort()
## get iteration number from iter dir
self.params['currentiter'] = int(alliters[-1][-2:])
self.params['iterdir'] = os.path.join(self.params['rundir'],alliters[-1])
self.params['currentcls'] = "classes%02i"%(self.params['currentiter'])
## go into last iteration directory
os.chdir(self.params['iterdir'])
self.params['alignedstack'] = os.path.abspath("mrastack")
if os.path.isfile(os.path.join(self.params['rundir'],self.params['currentcls']+".hed")):
p1 = os.path.join(self.params['rundir'],self.params['currentcls'])
p2 = os.path.join(self.params['iterdir'],self.params['currentcls'])
shutil.move(p1+".hed",p2+".hed")
shutil.move(p1+".img",p2+".img")
## sort the class averages
self.sortClassAverages()
### get particle information from last iteration
if self.params['mramethod']=='imagic':
partlist = self.readPartIMAGICFile()
else:
partlist = self.readPartEMANFile()
if self.params['msamethod']=='imagic':
partrefdict = self.imagicClassificationToDict()
else:
partrefdict = self.canClassificationToDict()
# move back to starting directory
os.chdir(self.params['rundir'])
# move aligned stack to current directory for appionweb
if not os.path.isfile("mrastack.hed"):
shutil.move(self.params['alignedstack']+".hed","mrastack.hed")
shutil.move(self.params['alignedstack']+".img","mrastack.img")
# rewrite header
if self.params['mramethod'] == "imagic" or self.params['msamethod'] == 'imagic':
imagicFilters.takeoverHeaders("mrastack",self.params['numpart'],self.workingboxsize)
# move actual averages to current directory
if self.params['msamethod']=='can':
if not os.path.isfile("classes_avg.hed"):
self.applySort()
# save actual class averages as refs in database
self.params['currentcls']="classes_avg"
### create an average mrc of final references
if not os.path.isfile("average.mrc"):
apStack.averageStack(stack=self.params['currentcls']+".hed")
self.dumpParameters()
### remove the filtered stack
apFile.removeStack(origstack)
### save to database
if self.params['commit'] is True:
self.insertRunIntoDatabase()
self.insertParticlesIntoDatabase(partlist, partrefdict)