def loopProcessImage(self, imgdata):
#creates self.peaktree and filterLoop sets self.filtarray
self.peaktree = filterLoop.FilterLoop.loopProcessImage(self, imgdata)
if self.params['background'] is False:
apDisplay.printMsg("Found " + str(len(self.peaktree)) +
" particles for " +
apDisplay.shortenImageName(imgdata['filename']))
self.stats['lastpeaks'] = len(self.peaktree)
if self.params['nojpegs'] is False:
if self.threadJpeg is True:
threading.Thread(target=apPeaks.createPeakJpeg,
args=(imgdata, self.peaktree, self.params,
self.filtarray)).start()
else:
apPeaks.createPeakJpeg(imgdata, self.peaktree, self.params,
self.filtarray)
elif self.params['background'] is False:
apDisplay.printWarning("Skipping JPEG creation")
if self.params['defocpair'] is True:
self.sibling, self.shiftpeak = apDefocalPairs.getShiftFromImage(
imgdata, self.params['sessionname'])
return
def _startLoop(self, imgdata):
"""
initilizes several parameters for a new image
and checks if it is okay to start processing image
"""
if self.params['parallel']:
if self.lockParallel(imgdata.dbid):
apDisplay.printMsg('%s locked by another parallel run in the rundir' % (apDisplay.shortenImageName(imgdata['filename'])))
return False
#calc images left
self.stats['imagesleft'] = self.stats['imagecount'] - self.stats['count']
#only if an image was processed last
if(self.stats['lastcount'] != self.stats['count']):
if self.params['background'] is False:
apDisplay.printColor( "\nStarting image %d ( skip:%d, remain:%d ) id:%d, file: %s"
%(self.stats['count'], self.stats['skipcount'], self.stats['imagesleft'],
imgdata.dbid, apDisplay.short(imgdata['filename']),),
"green")
elif self.stats['count'] % 80 == 0:
sys.stderr.write("\n")
self.stats['lastcount'] = self.stats['count']
if apDisplay.isDebugOn():
self._checkMemLeak()
# skip if image doesn't exist:
imgpath = os.path.join(imgdata['session']['image path'], imgdata['filename']+'.mrc')
if not os.path.isfile(imgpath):
apDisplay.printWarning(imgpath+" not found, skipping")
if self.params['parallel']:
self.unlockParallel(imgdata.dbid)
return False
# check to see if image has already been processed
if self._alreadyProcessed(imgdata):
if self.params['parallel']:
self.unlockParallel(imgdata.dbid)
return False
self.stats['waittime'] = 0
if self.reprocessImage(imgdata) is True:
if self.params['background'] is True:
sys.stderr.write(",")
else:
"""apDisplay.printMsg("reprocessing "+apDisplay.shortenImageName(imgdata['filename']))"""
else:
if self.params['background'] is True:
sys.stderr.write(".")
else:
"""apDisplay.printMsg("processing "+apDisplay.shortenImageName(imgdata['filename']))"""
return True
def loopProcessImage(self, imgdata):
#creates self.peaktree and filterLoop sets self.filtarray
self.peaktree = filterLoop.FilterLoop.loopProcessImage(self, imgdata)
if self.params['background'] is False:
apDisplay.printMsg("Found "+str(len(self.peaktree))+" particles for "
+apDisplay.shortenImageName(imgdata['filename']))
self.stats['lastpeaks'] = len(self.peaktree)
if self.params['nojpegs'] is False:
if self.threadJpeg is True:
threading.Thread(target=apPeaks.createPeakJpeg, args=(imgdata, self.peaktree, self.params, self.filtarray)).start()
else:
apPeaks.createPeakJpeg(imgdata, self.peaktree, self.params, self.filtarray)
elif self.params['background'] is False:
apDisplay.printWarning("Skipping JPEG creation")
if self.params['defocpair'] is True:
self.sibling, self.shiftpeak = apDefocalPairs.getShiftFromImage(imgdata, self.params['sessionname'])
return
class CatchUpFrameAlignmentLoop(appionScript.AppionScript):
#=======================
def setupParserOptions(self):
self.parser.add_option("--ddstackid",
dest="ddstackid",
type="int",
help="ID for dd frame stack run",
metavar="INT")
self.parser.add_option(
"--no-wait",
dest="wait",
default=True,
action="store_false",
help="Do not wait for frame stack to finish creation")
self.parser.add_option(
"-m",
"--mrclist",
dest="mrcnames",
help=
"List of mrc files to process, e.g. --mrclist=..003en,..002en,..006en",
metavar="MRCNAME")
self.parser.add_option("--gpuid",
dest="gpuid",
type="int",
default=0,
help="GPU device id used in gpu processing",
metavar="INT")
self.parser.add_option(
"--limit",
dest="limit",
type="int",
default=0,
help="Limit image processing to this number. 0 means no limit",
metavar="INT")
#=======================
def checkConflicts(self):
# make sure program exist
exename = 'dosefgpu_driftcorr'
driftcorrexe = subprocess.Popen(
"which " + exename, shell=True,
stdout=subprocess.PIPE).stdout.read().strip()
if not os.path.isfile(driftcorrexe):
apDisplay.printError('Drift correction program not available')
if not self.params['ddstackid']:
apDisplay.printMsg('Must have a ddstack run to catch up alignment')
# make sure ddstack exists
ddstackrun = appiondata.ApDDStackRunData().direct_query(
self.params['ddstackid'])
if ddstackrun:
apDisplay.printMsg('Found dd frame stack run')
# set self.rundata in this function because we may need it if rundir is not set in params
self.rundata = ddstackrun
else:
apDisplay.printError('DD Frame Stack id %d does not exist' %
self.params['ddstackid'])
if not self.rundata['params']['align']:
apDisplay.printError(
'DD Frame Stack id %d was not meant to be aligned' %
self.params['ddstackid'])
#=====================
def setRunDir(self):
self.params['rundir'] = self.rundata['path']['path']
#=======================
def onInit(self):
if 'sessionname' not in self.params.keys():
self.params['sessionname'] = leginondata.SessionData(
).direct_query(self.params['expid'])['name']
self.dd = apDDprocess.initializeDDFrameprocess(
self.params['sessionname'], self.params['wait'])
self.dd.setRunDir(self.params['rundir'])
# The gain/dark corrected ddstack is unlikely to be on local disk
if 'tempdir' not in self.params.keys():
self.dd.setTempDir()
else:
self.dd.setTempDir(self.params['tempdir'])
self.dd.setNewBinning(self.rundata['params']['bin'])
# Get the unfinished ddstack run parameters to apply them here
jobdata = apDatabase.getJobDataFromPathAndType(
self.rundata['path']['path'], 'makeddrawframestack')
self.ddstack_script_params = apScriptLog.getScriptParamValuesFromRunname(
self.rundata['runname'], self.rundata['path'], jobdata)
if 'no-keepstack' in self.ddstack_script_params.keys():
self.dd.setKeepStack(False)
self.dd.setDoseFDriftCorrOptions(self.ddstack_script_params)
# Give an unique lockname
self.setLockname('ddalign')
self.success_count = 0
def hasDDAlignedImagePair(self):
alignpairdata = self.dd.getAlignImagePairData(self.rundata,
query_source=True)
return bool(alignpairdata)
#=======================
def processImage(self, imgdata):
# initialize aligned_imagedata as if not aligned
self.aligned_imagedata = None
# need to avoid non-frame saved image for proper caching
if imgdata is None or imgdata['camera']['save frames'] != True:
apDisplay.printWarning('%s skipped for no-frame-saved\n ' %
imgdata['filename'])
return
### set processing image
try:
self.dd.setImageData(imgdata, ignore_raw=True)
except Exception, e:
apDisplay.printWarning(e.message)
return
## various ways to skip the image
if self.hasDDAlignedImagePair():
apDisplay.printWarning(
'aligned image %d from this run is already in the database. Skipped....'
% imgdata.dbid)
return
if self.lockParallel(imgdata.dbid):
apDisplay.printMsg(
'%s locked by another parallel run in the rundir' %
(apDisplay.shortenImageName(imgdata['filename'])))
return
# This will wait for the stack to finish gain correction
if not self.dd.isReadyForAlignment():
apDisplay.printWarning(
'unaligned frame stack not created. Skipped....')
self.unlockParallel(imgdata.dbid)
return
# set align parameters for the image
framelist = self.dd.getFrameList(self.ddstack_script_params)
self.dd.setAlignedSumFrameList(framelist)
self.dd.setGPUid(self.params['gpuid'])
self.dd.setAlignedCameraEMData()
if not self.dd.hasBadPixels():
# use GPU to do flat field correction if no bad pixel/col/rows
self.dd.setUseGPUFlat(True)
self.dd.gainCorrectAndAlignFrameStack()
else:
self.dd.alignCorrectedFrameStack()
if os.path.isfile(self.dd.aligned_stackpath):
if 'alignlabel' not in self.ddstack_script_params.keys(
) or not self.ddstack_script_params['alignlabel']:
# appion script params may not have included alignlabel
self.ddstack_script_params['alignlabel'] = 'a'
self.aligned_imagedata = self.dd.makeAlignedImageData(
alignlabel=self.ddstack_script_params['alignlabel'])
apDisplay.printMsg(
' Replacing unaligned stack with the aligned one....')
apFile.removeFile(self.dd.framestackpath)
shutil.move(self.dd.aligned_stackpath, self.dd.framestackpath)
self.success_count += 1
self.unlockParallel(imgdata.dbid)
def _startLoop(self, imgdata):
"""
initilizes several parameters for a new image
and checks if it is okay to start processing image
"""
if self.params['parallel']:
if self.lockParallel(imgdata.dbid):
apDisplay.printMsg(
'%s locked by another parallel run in the rundir' %
(apDisplay.shortenImageName(imgdata['filename'])))
return False
#calc images left
self.stats[
'imagesleft'] = self.stats['imagecount'] - self.stats['count']
#only if an image was processed last
if (self.stats['lastcount'] != self.stats['count']):
if self.params['background'] is False:
apDisplay.printColor(
"\nStarting image %d ( skip:%d, remain:%d ) id:%d, file: %s"
% (
self.stats['count'],
self.stats['skipcount'],
self.stats['imagesleft'],
imgdata.dbid,
apDisplay.short(imgdata['filename']),
), "green")
elif self.stats['count'] % 80 == 0:
sys.stderr.write("\n")
self.stats['lastcount'] = self.stats['count']
self._checkMemLeak()
# skip if image doesn't exist:
imgpath = os.path.join(imgdata['session']['image path'],
imgdata['filename'] + '.mrc')
if not os.path.isfile(imgpath):
apDisplay.printWarning(imgpath + " not found, skipping")
if self.params['parallel']:
self.unlockParallel(imgdata.dbid)
return False
# check to see if image has already been processed
if self._alreadyProcessed(imgdata):
if self.params['parallel']:
self.unlockParallel(imgdata.dbid)
return False
self.stats['waittime'] = 0
if self.reprocessImage(imgdata) is True:
if self.params['background'] is True:
sys.stderr.write(",")
else:
"""apDisplay.printMsg("reprocessing "+apDisplay.shortenImageName(imgdata['filename']))"""
else:
if self.params['background'] is True:
sys.stderr.write(".")
else:
"""apDisplay.printMsg("processing "+apDisplay.shortenImageName(imgdata['filename']))"""
return True