def commitToDatabase(self, imgdata):
"""
Uses the appionLoop commit
"""
imgassess = apDatabase.getImgCompleteStatus(imgdata)
tiltdata = apTiltPair.getTiltPair(imgdata)
msg = not self.params['background']
if tiltdata is None:
if imgassess is not False:
apDisplay.printColor("\nrejecting unpaired image: "+apDisplay.short(imgdata['filename']), "red")
apDatabase.insertImgAssessmentStatus(imgdata, self.params['runid'], False, msg=msg)
self.reject+=1
return
if self.params['background'] is False:
apDisplay.printMsg("tiltpair: "+apDisplay.short(tiltdata['filename']))
tiltassess = apDatabase.getImgCompleteStatus(tiltdata)
if imgassess is False or tiltassess is False:
if imgassess is not False:
apDisplay.printColor("\nrejecting bad tilt images: "+apDisplay.short(imgdata['filename']), "magenta")
if tiltassess is not False:
apDisplay.printColor("\nrejecting bad tilt images: "+apDisplay.short(tiltdata['filename']), "magenta")
apDatabase.insertImgAssessmentStatus(imgdata, self.params['runid'], False, msg=msg)
apDatabase.insertImgAssessmentStatus(tiltdata, self.params['runid'], False, msg=msg)
self.reject+=2
if self.params['background'] is False:
print "Assessment:", imgassess, tiltassess
return
def insertShift(imgdata, siblingdata, peak):
if not siblingdata or not peak:
apDisplay.printWarning("No sibling or peak found. No database insert")
return False
shiftq = appiondata.ApImageTransformationData()
shiftq["image1"] = imgdata
shiftdata = shiftq.query()
if shiftdata:
apDisplay.printWarning("Shift values already in database")
return False
shiftq["image2"] = siblingdata
shiftq["shiftx"] = peak["shift"][1]
shiftq["shifty"] = peak["shift"][0]
shiftq["scale"] = peak["scalefactor"]
if not peak["subfailed"]:
shiftq["correlation"] = peak["subpixel peak value"]
else:
shiftq["correlation"] = 0.0
apDisplay.printMsg(
"Inserting shift beteween "
+ apDisplay.short(imgdata["filename"])
+ " and "
+ apDisplay.short(siblingdata["filename"])
+ " into database"
)
shiftq.insert()
return True
def commitToDatabase(self, imgdata):
"""
Uses the appionLoop commit
"""
imgassess = apDatabase.getImgCompleteStatus(imgdata)
tiltdata = apTiltPair.getTiltPair(imgdata)
msg = not self.params['background']
if tiltdata is None:
if imgassess is not False:
apDisplay.printColor("\nrejecting unpaired image: "+apDisplay.short(imgdata['filename']), "red")
apDatabase.insertImgAssessmentStatus(imgdata, self.params['runid'], False, msg=msg)
self.reject+=1
return
if self.params['background'] is False:
apDisplay.printMsg("tiltpair: "+apDisplay.short(tiltdata['filename']))
tiltassess = apDatabase.getImgCompleteStatus(tiltdata)
if imgassess is False or tiltassess is False:
if imgassess is not False:
apDisplay.printColor("\nrejecting bad tilt images: "+apDisplay.short(imgdata['filename']), "magenta")
if tiltassess is not False:
apDisplay.printColor("\nrejecting bad tilt images: "+apDisplay.short(tiltdata['filename']), "magenta")
apDatabase.insertImgAssessmentStatus(imgdata, self.params['runid'], False, msg=msg)
apDatabase.insertImgAssessmentStatus(tiltdata, self.params['runid'], False, msg=msg)
self.reject+=2
if self.params['background'] is False:
print "Assessment:", imgassess, tiltassess
return
def rejectAceInfo(self, imgdata):
shortname = apDisplay.short(imgdata["filename"])
### get best defocus value
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata)
if ctfvalue is None:
if self.params["noace"] is True:
apDisplay.printColor("\nrejecting no ACE values: " + apDisplay.short(imgdata["filename"]), "yellow")
return False
else:
# apDisplay.printWarning("skipping no ACE values for "+apDisplay.short(imgdata['filename']))
return True
### check that CTF estimation is above confidence threshold
if self.params["ctfcutoff"] and conf < self.params["ctfcutoff"]:
apDisplay.printColor(
"\nrejecting below CTF cutoff: "
+ apDisplay.short(imgdata["filename"])
+ " conf="
+ str(round(conf, 3)),
"cyan",
)
return False
### defocus should be in negative meters
if ctfvalue["defocus2"] is not None and ctfvalue["defocus1"] != ctfvalue["defocus2"]:
defocus = (ctfvalue["defocus1"] + ctfvalue["defocus2"]) / 2.0
else:
defocus = ctfvalue["defocus1"]
defocus = -1.0 * abs(defocus)
### skip micrograph that have defocus above or below min & max defocus levels
if self.params["mindefocus"] and defocus > self.params["mindefocus"]:
apDisplay.printColor(
shortname
+ " defocus ("
+ str(round(defocus * 1e6, 2))
+ " um) is less than mindefocus ("
+ str(self.params["mindefocus"] * 1e6)
+ " um)\n",
"cyan",
)
return False
if self.params["maxdefocus"] and defocus < self.params["maxdefocus"]:
apDisplay.printColor(
shortname
+ " defocus ("
+ str(round(defocus * 1e6, 2))
+ " um) is greater than maxdefocus ("
+ str(self.params["maxdefocus"] * 1e6)
+ " um)\n",
"cyan",
)
return False
return True
def getShift(imgdata1, imgdata2):
# assumes images are square
print "Finding shift between", apDisplay.short(imgdata1["filename"]), "and", apDisplay.short(imgdata2["filename"])
dimension1 = imgdata1["camera"]["dimension"]["x"]
binning1 = imgdata1["camera"]["binning"]["x"]
dimension2 = imgdata2["camera"]["dimension"]["x"]
binning2 = imgdata2["camera"]["binning"]["x"]
finalsize = 512
# test to make sure images are at same mag
if imgdata1["scope"]["magnification"] != imgdata2["scope"]["magnification"]:
apDisplay.printWarning("Defocus pairs are at different magnifications, so shift can't be calculated.")
return None
# test to see if images capture the same area
if (dimension1 * binning1) != (dimension2 * binning2):
apDisplay.printWarning("Defocus pairs do not capture the same imaging area, so shift can't be calculated.")
return None
# images must not be less than finalsize (currently 512) pixels. This is arbitrary but for good reason
if dimension1 < finalsize or dimension2 < finalsize:
apDisplay.printWarning("Images must be greater than " + finalsize + " pixels to calculate shift.")
return None
# low pass filter 2 images to twice the final pixelsize BEFORE binning
shrinkfactor1 = dimension1 / finalsize
shrinkfactor2 = dimension2 / finalsize
binned1 = apImage.filterImg(imgdata1["image"], 1.0, shrinkfactor1 * 2)
binned2 = apImage.filterImg(imgdata2["image"], 1.0, shrinkfactor2 * 2)
# now bin 2 images
binned1 = apImage.binImg(binned1, shrinkfactor1)
binned2 = apImage.binImg(binned2, shrinkfactor2)
### fix for non-square images, correlation fails on non-square images
mindim = min(binned1.shape)
binned1 = binned1[:mindim, :mindim]
binned2 = binned2[:mindim, :mindim]
### use phase correlation, performs better than cross
pc = correlator.phase_correlate(binned1, binned2)
apImage.arrayToMrc(pc, "phaseCorrelate.mrc")
### find peak, filtering to 10.0 helps
peak = peakfinder.findSubpixelPeak(pc, lpf=10.0)
subpixpeak = peak["subpixel peak"]
shift = correlator.wrap_coord(subpixpeak, pc.shape)
peak["scalefactor"] = dimension2 / float(dimension1)
# print shift[0]*shrinkfactor1, shift[1]*shrinkfactor1
xshift = int(round(shift[0] * shrinkfactor1))
yshift = int(round(shift[1] * shrinkfactor1))
peak["shift"] = numpy.array((xshift, yshift))
apDisplay.printMsg("Determined shifts: %f %f" % (peak["shift"][0], peak["shift"][1]))
# print peak['shift']
# sys.exit(1)
return peak
def rejectTiltPairs(self, imgdata):
tiltdata = apTiltPair.getTiltPair(imgdata)
if tiltdata is None:
apDisplay.printColor("\nrejecting unpaired image: "+apDisplay.short(imgdata['filename']), "red")
return False
tiltassess = apDatabase.getImgCompleteStatus(tiltdata)
if tiltassess is False:
apDisplay.printColor("\nrejecting bad tilt images: "+apDisplay.short(imgdata['filename']), "magenta")
return False
return True
def getDefocPairParticles(imgdata, selectionid, particlelabel=None):
### get defocal pair
if imgdata['preset'] is not None:
defimgdata = apDefocalPairs.getDefocusPair(imgdata)
else:
# For pairs from Leginon Manual Application
defimgdata = apDefocalPairs.getManualDefocusPair(imgdata)
if defimgdata is None:
apDisplay.printWarning(
"Could not find defocal pair for image %s (id %d)" %
(apDisplay.short(imgdata['filename']), imgdata.dbid))
return ([], {'shiftx': 0, 'shifty': 0, 'scale': 1})
apDisplay.printMsg(
"Found defocus pair %s (id %d) for image %s (id %d)" %
(apDisplay.short(defimgdata['filename']), defimgdata.dbid,
apDisplay.short(imgdata['filename']), imgdata.dbid))
### get particles
partq = appiondata.ApParticleData()
partq['image'] = defimgdata
partq['selectionrun'] = appiondata.ApSelectionRunData.direct_query(
selectionid)
if particlelabel is not None:
partq['label'] = particlelabel
partdatas = partq.query()
apDisplay.printMsg("Found %d particles for defocal pair %s (id %d)" % (
len(partdatas),
apDisplay.short(defimgdata['filename']),
defimgdata.dbid,
))
if len(partdatas) == 0:
return ([], {'shiftx': 0, 'shifty': 0, 'scale': 1})
### get shift information
has_tried = False
while True:
shiftq = appiondata.ApImageTransformationData()
shiftq['image1'] = defimgdata
shiftdatas = shiftq.query()
if shiftdatas:
shiftdata = shiftdatas[0]
apDisplay.printMsg(
"Shifting particles by %.1f,%.1f (%d X)" %
(shiftdata['shiftx'], shiftdata['shifty'], shiftdata['scale']))
break
else:
if has_tried == True:
apDisplay.printError(
"No shift inserted to database after one try")
apDisplay.printMsg("Calculating shift....")
shiftpeak = apDefocalPairs.getShift(defimgdata, imgdata)
apDefocalPairs.insertShift(defimgdata, imgdata, shiftpeak)
has_tried = True
return (partdatas, shiftdata)
def rejectTiltPairs(self, imgdata):
tiltdata = apTiltPair.getTiltPair(imgdata)
if tiltdata is None:
apDisplay.printColor(
"\nrejecting unpaired image: " +
apDisplay.short(imgdata['filename']), "red")
return False
tiltassess = apDatabase.getImgCompleteStatus(tiltdata)
if tiltassess is False:
apDisplay.printColor(
"\nrejecting bad tilt images: " +
apDisplay.short(imgdata['filename']), "magenta")
return False
return True
def validateAndInsertCTFData(imgdata,
ctfvalues,
rundata,
rundir,
fftpath=None,
fftfreq=None):
"""
function to insert CTF values in database
"""
apDisplay.printMsg("Committing ctf parameters for " +
apDisplay.short(imgdata['filename']) + " to database")
if ctfvalues is None or not 'defocus2' in ctfvalues:
apDisplay.printWarning("No ctf values")
return False
### convert to common convention
ctfvalues = convertDefociToConvention(ctfvalues)
### check to make sure parameters are valid
isvalid = checkParams(ctfvalues)
if isvalid is False:
apDisplay.printWarning("Bad CTF values, insert but not create images")
### run the main CTF display program
opimagedir = os.path.join(rundir, "opimages")
if isvalid is True:
oldctfvalues = ctfvalues.copy()
ctfvalues = runCTFdisplayTools(imgdata, ctfvalues, opimagedir, fftpath,
fftfreq)
# check if image creation failed
if ctfvalues is None:
ctfvalues = oldctfvalues
### clean rundir from all entries:
if not rundir.endswith("/"):
rundir += "/"
for key in ctfvalues.keys():
if isinstance(ctfvalues[key],
str) and ctfvalues[key].startswith(rundir):
ctfvalues[key] = ctfvalues[key].replace(rundir, "")
### time to insert
ctfq = appiondata.ApCtfData()
ctfq['acerun'] = rundata
ctfq['image'] = imgdata
if debug is True:
apDisplay.printMsg("CTF data values")
print ctfvalues
for key in ctfq.keys():
if key in ctfvalues:
ctfq[key] = ctfvalues[key]
if debug is True:
apDisplay.printMsg("%s :: %s" % (key, ctfvalues[key]))
elif debug is True:
apDisplay.printMsg("SKIPPING %s :: %s" % (key, ctfvalues[key]))
ctfdb.printCtfData(ctfq)
ctfq.insert()
return
def start(self):
imgfilename2peaklist = self.readFileToPeakTree()
imglist = imgfilename2peaklist.keys()
apDisplay.printMsg("Getting image data from database")
imgtree = apDatabase.getSpecificImagesFromDB(imglist, self.sessiondata)
if imgtree[0]['session']['name'] != self.sessiondata['name']:
apDisplay.printError("Session and Image do not match " +
imgtree[0]['filename'])
### insert params for manual picking
rundata = self.insertManualParams()
# upload Particles
for imgdata in imgtree:
### check session
if imgdata['session']['name'] != self.sessiondata['name']:
apDisplay.printError("Session and Image do not match " +
imgdata['filename'])
peaktree = imgfilename2peaklist[imgdata['filename']]
apDisplay.printMsg(
"%d particles for image %s" %
(len(peaktree), apDisplay.short(imgdata['filename'])))
if self.params['commit'] is True:
apParticle.insertParticlePeaks(peaktree,
imgdata,
self.params['runname'],
msg=True)
def processParticles(self, imgdata, partdatas, shiftdata):
# need to avoid non-frame saved image for proper caching
if imgdata is None or imgdata['camera']['save frames'] != True:
apDisplay.printWarning('%s skipped for no-frame-saved\n ' %
imgdata['filename'])
return
### first remove any existing files
rundir = self.params['rundir']
imgname = imgdata['filename']
moviegrouppath = os.path.join(rundir,
'%s_*.%s' % (imgname, self.movieformat))
apFile.removeFile(moviegrouppath)
shortname = apDisplay.short(imgdata['filename'])
apFile.removeFile(os.path.join(rundir, '%s*.jpg' % (shortname)))
### set processing image
try:
self.dd.setImageData(imgdata)
except Exception, e:
raise
apDisplay.printWarning('%s: %s' % (e.__class__.__name__, e))
return
def processImage(self, imgdata): self.ctfvalues = None fftpath = os.path.join(self.powerspecdir, apDisplay.short(imgdata['filename'])+'.powerspec.mrc') self.processAndSaveFFT(imgdata, fftpath) self.runRefineCTF(imgdata, fftpath) apFile.removeFile(fftpath) return
def processParticles(self, imgdata, partdatas, shiftdata):
self.shortname = apDisplay.short(imgdata['filename'])
### if only selected points along helix,
### fill in points with helical step
if self.params['helicalstep']:
apix = apDatabase.getPixelSize(imgdata)
partdatas = self.fillWithHelicalStep(partdatas, apix)
### run batchboxer
self.boxedpartdatas, self.imgstackfile, self.partmeantree = self.boxParticlesFromImage(imgdata, partdatas, shiftdata)
if self.boxedpartdatas is None:
self.stats['lastpeaks'] = 0
apDisplay.printWarning("no particles were boxed from "+self.shortname+"\n")
self.badprocess = True
return None
self.stats['lastpeaks'] = len(self.boxedpartdatas)
apDisplay.printMsg("do not break function now otherwise it will corrupt stack")
#time.sleep(1.0)
### merge image particles into big stack
totalpart = self.mergeImageStackIntoBigStack(self.imgstackfile, imgdata)
### create a stack average every so often
if self.stats['lastpeaks'] > 0:
totalPartices = self.existingParticleNumber+self.stats['peaksum']+self.stats['lastpeaks']
logpeaks = math.log(totalPartices)
if logpeaks > self.logpeaks:
self.logpeaks = math.ceil(logpeaks)
numpeaks = math.ceil(math.exp(self.logpeaks))
apDisplay.printMsg("writing averaging stack, next average at %d particles"%(numpeaks))
mrc.write(self.summedParticles/float(totalPartices), "average.mrc")
return totalpart
def getCtfValueForCtfRunId(imgdata, ctfrunid=None, msg=False):
"""
takes an image and get the ctf value for that image for the specified ctf estimation run
specified methods can be: ace2 or ctffind
"""
if ctfrunid is None:
apDisplay.printError("This function requires a ctf_estimation_runid")
ctfrundata = appiondata.ApAceRunData.direct_query(ctfrunid)
if ctfrundata is None:
apDisplay.printError("ctf_estimation_runid not found")
### get all ctf values
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfq['acerun'] = ctfrundata
ctfvalues = ctfq.query()
### check if it has values
if ctfvalues is None:
return None
### find the value
ctfdata = None
if len(ctfvalues) == 1:
if msg is True:
printCtfData(ctfvalues[0])
return ctfvalues[0]
elif len(ctfvalues) > 1:
apDisplay.printWarning("more than one run found for ctfrunid %d and image %s"
%(ctfrunid, apDisplay.short(imgdata['filename'])))
return ctfvalues[-1]
return None
def checkCtfParams(self, imgdata):
shortname = apDisplay.short(imgdata['filename'])
ctfvalue, conf = self.getCtfValueConfidenceForImage(imgdata, False)
### check if we have values and if we care
if ctfvalue is None:
return not self.checkRequireCtf()
### check that CTF estimation is above confidence threshold
if self.params['ctfcutoff'] and conf < self.params['ctfcutoff']:
#apDisplay.printColor(shortname+" is below CTF threshold (conf="+str(round(conf,3))+")\n","cyan")
return False
### check resolution requirement for CTF fit at 0.8 threshold
if self.params['ctfres80'] is not None:
if not 'resolution_80_percent' in ctfvalue.keys():
return False
if ctfvalue['resolution_80_percent'] > self.params['ctfres80']:
### resolution is too poor
return False
### check resolution requirement for CTF fit at 0.5 threshold
if self.params['ctfres50'] is not None:
if not 'resolution_50_percent' in ctfvalue.keys():
return False
if ctfvalue['resolution_50_percent'] > self.params['ctfres50']:
### resolution is too poor
return False
### get best defocus value
### defocus should be in negative meters
if ctfvalue['defocus2'] is not None and ctfvalue[
'defocus1'] != ctfvalue['defocus2']:
defocus = (ctfvalue['defocus1'] + ctfvalue['defocus2']) / 2.0
else:
defocus = ctfvalue['defocus1']
defocus = -1.0 * abs(defocus)
### assume defocus values are ALWAYS negative but mindefocus is greater than maxdefocus
if self.params['mindefocus']:
self.params['mindefocus'] = -abs(self.params['mindefocus'])
if self.params['maxdefocus']:
self.params['maxdefocus'] = -abs(self.params['maxdefocus'])
if self.params['mindefocus'] and self.params['maxdefocus']:
if self.params['maxdefocus'] > self.params['mindefocus']:
mindef = self.params['mindefocus']
maxdef = self.params['maxdefocus']
self.params['mindefocus'] = maxdef
self.params['maxdefocus'] = mindef
### skip micrograph that have defocus above or below min & max defocus levels
if self.params['mindefocus'] and defocus > self.params['mindefocus']:
#apDisplay.printColor(shortname+" defocus ("+str(round(defocus*1e6,2))+\
# " um) is less than mindefocus ("+str(self.params['mindefocus']*1e6)+" um)\n","cyan")
return False
if self.params['maxdefocus'] and defocus < self.params['maxdefocus']:
#apDisplay.printColor(shortname+" defocus ("+str(round(defocus*1e6,2))+\
# " um) is greater than maxdefocus ("+str(self.params['maxdefocus']*1e6)+" um)\n","cyan")
return False
return True
def _startLoop(self, info):
"""
appionLoop OVERRIDE
initilizes several parameters for a new image
and checks if it is okay to start processing image
"""
if info is None:
self.stats['lastimageskipped'] = True
self.stats['skipcount'] += 1
return False
name = info['filename']
# check to see if image of the same name is already in leginon
imgq = leginon.leginondata.AcquisitionImageData(session=self.session, filename=name)
results = imgq.query(readimages=False)
if results:
apDisplay.printWarning("File %s.mrc exists at the destination" % name)
apDisplay.printWarning("Skip Uploading")
self.stats['lastimageskipped'] = True
self.stats['skipcount'] += 1
return False
#calc images left
self.stats['imagesleft'] = self.stats['imagecount'] - self.stats['count']
#only if an image was processed last
if(self.stats['lastcount'] != self.stats['count']):
apDisplay.printColor( "\nStarting image "+str(self.stats['count'])\
+" ( skip:"+str(self.stats['skipcount'])+", remain:"\
+str(self.stats['imagesleft'])+" ) file: "\
+apDisplay.short(name), "green")
self.stats['lastcount'] = self.stats['count']
self._checkMemLeak()
# check to see if image has already been processed
if self._alreadyProcessed(info):
return False
self.stats['waittime'] = 0
return True
def checkCtfParams(self, imgdata):
shortname = apDisplay.short(imgdata['filename'])
ctfvalue, conf = self.getCtfValueConfidenceForImage(imgdata,False)
### check if we have values and if we care
if ctfvalue is None:
return not self.checkRequireCtf()
### check that CTF estimation is above confidence threshold
if self.params['ctfcutoff'] and conf < self.params['ctfcutoff']:
#apDisplay.printColor(shortname+" is below CTF threshold (conf="+str(round(conf,3))+")\n","cyan")
return False
### check resolution requirement for CTF fit at 0.8 threshold
if self.params['ctfres80'] is not None:
if not 'resolution_80_percent' in ctfvalue.keys():
return False
if ctfvalue['resolution_80_percent'] > self.params['ctfres80']:
### resolution is too poor
return False
### check resolution requirement for CTF fit at 0.5 threshold
if self.params['ctfres50'] is not None:
if not 'resolution_50_percent' in ctfvalue.keys():
return False
if ctfvalue['resolution_50_percent'] > self.params['ctfres50']:
### resolution is too poor
return False
### get best defocus value
### defocus should be in negative meters
if ctfvalue['defocus2'] is not None and ctfvalue['defocus1'] != ctfvalue['defocus2']:
defocus = (ctfvalue['defocus1'] + ctfvalue['defocus2'])/2.0
else:
defocus = ctfvalue['defocus1']
defocus = -1.0*abs(defocus)
### assume defocus values are ALWAYS negative but mindefocus is greater than maxdefocus
if self.params['mindefocus']:
self.params['mindefocus'] = -abs( self.params['mindefocus'] )
if self.params['maxdefocus']:
self.params['maxdefocus'] = -abs( self.params['maxdefocus'] )
if self.params['mindefocus'] and self.params['maxdefocus']:
if self.params['maxdefocus'] > self.params['mindefocus']:
mindef = self.params['mindefocus']
maxdef = self.params['maxdefocus']
self.params['mindefocus'] = maxdef
self.params['maxdefocus'] = mindef
### skip micrograph that have defocus above or below min & max defocus levels
if self.params['mindefocus'] and defocus > self.params['mindefocus']:
#apDisplay.printColor(shortname+" defocus ("+str(round(defocus*1e6,2))+\
# " um) is less than mindefocus ("+str(self.params['mindefocus']*1e6)+" um)\n","cyan")
return False
if self.params['maxdefocus'] and defocus < self.params['maxdefocus']:
#apDisplay.printColor(shortname+" defocus ("+str(round(defocus*1e6,2))+\
# " um) is greater than maxdefocus ("+str(self.params['maxdefocus']*1e6)+" um)\n","cyan")
return False
return True
def loopCleanUp(self, imgdata):
### last remove any existing boxed files, reset global params
shortname = apDisplay.short(imgdata['filename'])
shortfileroot = os.path.join(self.params['rundir'], shortname)
rmfiles = glob.glob(shortfileroot + "*")
if not self.params['keepall']:
for rmfile in rmfiles:
apFile.removeFile(rmfile)
def processImage(self, imgdata):
self.ctfvalues = None
fftpath = os.path.join(
self.powerspecdir,
apDisplay.short(imgdata['filename']) + '.powerspec.mrc')
self.processAndSaveFFT(imgdata, fftpath)
self.runRefineCTF(imgdata, fftpath)
return
def loopCleanUp(self,imgdata): ### last remove any existing boxed files, reset global params shortname = apDisplay.short(imgdata['filename']) shortfileroot = os.path.join(self.params['rundir'], shortname) rmfiles = glob.glob(shortfileroot+"*") if not self.params['keepall']: for rmfile in rmfiles: apFile.removeFile(rmfile)
def getDefocPairParticles(imgdata, selectionid, particlelabel=None):
### get defocal pair
if imgdata['preset'] is not None:
defimgdata = apDefocalPairs.getDefocusPair(imgdata)
else:
# For pairs from Leginon Manual Application
defimgdata = apDefocalPairs.getManualDefocusPair(imgdata)
if defimgdata is None:
apDisplay.printWarning("Could not find defocal pair for image %s (id %d)"
%(apDisplay.short(imgdata['filename']), imgdata.dbid))
return ([], {'shiftx':0, 'shifty':0, 'scale':1})
apDisplay.printMsg("Found defocus pair %s (id %d) for image %s (id %d)"
%(apDisplay.short(defimgdata['filename']), defimgdata.dbid, apDisplay.short(imgdata['filename']), imgdata.dbid))
### get particles
partq = appiondata.ApParticleData()
partq['image'] = defimgdata
partq['selectionrun'] = appiondata.ApSelectionRunData.direct_query(selectionid)
if particlelabel is not None:
partq['label'] = particlelabel
partdatas = partq.query()
apDisplay.printMsg("Found %d particles for defocal pair %s (id %d)"
%(len(partdatas), apDisplay.short(defimgdata['filename']), defimgdata.dbid,))
if len(partdatas) == 0:
return ([], {'shiftx':0, 'shifty':0, 'scale':1})
### get shift information
has_tried = False
while True:
shiftq = appiondata.ApImageTransformationData()
shiftq['image1'] = defimgdata
shiftdatas = shiftq.query()
if shiftdatas:
shiftdata = shiftdatas[0]
apDisplay.printMsg("Shifting particles by %.1f,%.1f (%d X)"
%(shiftdata['shiftx'], shiftdata['shifty'], shiftdata['scale']))
break
else:
if has_tried == True:
apDisplay.printError("No shift inserted to database after one try")
apDisplay.printMsg("Calculating shift....")
shiftpeak = apDefocalPairs.getShift(defimgdata, imgdata)
apDefocalPairs.insertShift(defimgdata, imgdata, shiftpeak)
has_tried = True
return (partdatas, shiftdata)
def spiderToArray(filename, msg=True):
"""
takes a numpy and writes a SPIDER image
"""
numer = spider.read(filename)
if msg is True:
apDisplay.printMsg("reading SPIDER image: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
return numer
def spiderToArray(filename, msg=True):
"""
takes a numpy and writes a SPIDER image
"""
numer = spider.read(filename)
if msg is True:
apDisplay.printMsg("reading SPIDER image: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
return numer
def mrcToArray(filename, msg=True):
"""
takes a numpy and writes a Mrc
"""
numer = mrc.read(filename)
if msg is True:
apDisplay.printMsg("reading MRC: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
return numer
def mrcToArray(filename, msg=True):
"""
takes a numpy and writes a Mrc
"""
numer = mrc.read(filename)
if msg is True:
apDisplay.printMsg("reading MRC: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
return numer
def rejectImage(self, imgdata): shortname = apDisplay.short(imgdata['filename']) if self.params['mag']: if not apDatabase.checkMag(imgdata, self.params['mag']): apDisplay.printColor(shortname+" was not at the specific magnification","cyan") return False return True
def arrayToMrc(numer, filename, msg=True):
"""
takes a numpy and writes a Mrc
"""
#numer = numpy.asarray(numer, dtype=numpy.float32)
if msg is True:
apDisplay.printMsg("writing MRC: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
mrc.write(numer, filename)
return
def arrayToSpider(numer, filename, msg=True):
"""
takes a numpy and writes a SPIDER imag
"""
#numer = numpy.asarray(numer, dtype=numpy.float32)
if msg is True:
apDisplay.printMsg("writing SPIDER image: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
spider.write(numer, filename)
return
def rejectAceInfo(self, imgdata):
shortname = apDisplay.short(imgdata['filename'])
### get best defocus value
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata)
if ctfvalue is None:
if self.params['noace'] is True:
apDisplay.printColor(
"\nrejecting no ACE values: " +
apDisplay.short(imgdata['filename']), "yellow")
return False
else:
#apDisplay.printWarning("skipping no ACE values for "+apDisplay.short(imgdata['filename']))
return True
### check that CTF estimation is above confidence threshold
if self.params['ctfcutoff'] and conf < self.params['ctfcutoff']:
apDisplay.printColor(
"\nrejecting below CTF cutoff: " +
apDisplay.short(imgdata['filename']) + " conf=" +
str(round(conf, 3)), "cyan")
return False
### defocus should be in negative meters
if ctfvalue['defocus2'] is not None and ctfvalue[
'defocus1'] != ctfvalue['defocus2']:
defocus = (ctfvalue['defocus1'] + ctfvalue['defocus2']) / 2.0
else:
defocus = ctfvalue['defocus1']
defocus = -1.0 * abs(defocus)
### skip micrograph that have defocus above or below min & max defocus levels
if self.params['mindefocus'] and defocus > self.params['mindefocus']:
apDisplay.printColor(shortname+" defocus ("+str(round(defocus*1e6,2))+\
" um) is less than mindefocus ("+str(self.params['mindefocus']*1e6)+" um)\n","cyan")
return False
if self.params['maxdefocus'] and defocus < self.params['maxdefocus']:
apDisplay.printColor(shortname+" defocus ("+str(round(defocus*1e6,2))+\
" um) is greater than maxdefocus ("+str(self.params['maxdefocus']*1e6)+" um)\n","cyan")
return False
return True
def arrayToSpider(numer, filename, msg=True):
"""
takes a numpy and writes a SPIDER imag
"""
#numer = numpy.asarray(numer, dtype=numpy.float32)
if msg is True:
apDisplay.printMsg("writing SPIDER image: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
spider.write(numer, filename)
return
def arrayToMrc(numer, filename, msg=True):
"""
takes a numpy and writes a Mrc
"""
#numer = numpy.asarray(numer, dtype=numpy.float32)
if msg is True:
apDisplay.printMsg("writing MRC: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
mrc.write(numer, filename)
return
def rejectImage(self, imgdata):
shortname = apDisplay.short(imgdata['filename'])
if self.params['mag']:
if not apDatabase.checkMag(imgdata, self.params['mag']):
apDisplay.printColor(
shortname + " was not at the specific magnification",
"cyan")
return False
return True
def getBestTiltTransform(imgdata):
"""
For a given image get the best tilt transform
parameters in the database
"""
transdatas = []
## case 1: check if imgdata is image1
transq = appiondata.ApImageTiltTransformData()
transq['image1'] = imgdata
transdatas1 = transq.query()
if transdatas1:
transdatas.extend(transdatas1)
## case 2: check if imgdata is image2
transq = appiondata.ApImageTiltTransformData()
transq['image2'] = imgdata
transdatas2 = transq.query()
if transdatas2:
transdatas.extend(transdatas2)
### case 1: no transformation
if len(transdatas) == 0:
apDisplay.printWarning("no transformation found for image %s" %
(apDisplay.short(imgdata['filename'])))
return None
### case 2: single transformation, no sorting required
if len(transdatas) == 1:
apDisplay.printMsg(
"single transformation found with rmsd %.2f for image %s" %
(transdatas[0]['rmsd'], apDisplay.short(imgdata['filename'])))
return transdatas[0]
### case 3: more than one transformation, get one with lowest rmsd
besttransdata = transdatas[0]
bestrmsd = transdatas[0]['rmsd']
for transdata in transdatas:
if transdata['rmsd'] > 0 and transdata['rmsd'] < bestrmsd:
besttransdata = transdata
bestrmsd = transdata['rmsd']
apDisplay.printMsg(
"returning best transformation with rmsd %.2f for image %s" %
(bestrmsd, apDisplay.short(imgdata['filename'])))
return besttransdata
def processImage(self, imgdata):
### reset global value
self.imgassess = True
### get initial assessment
imgassess = apDatabase.getImgAssessmentStatus(imgdata)
if imgassess is False:
return
if imgassess is None:
imgassess = True
### hidden image
imgview = apDatabase.getImgViewerStatus(imgdata)
if imgview is False:
apDisplay.printColor(
"\nrejecting hidden image: " +
apDisplay.short(imgdata['filename']), "green")
self.imgassess = False
return
### tilt pair stuff
if imgassess is not False and self.params['notiltpairs'] is True:
imgassess = self.rejectTiltPairs(imgdata)
### picking stuff
if imgassess is not False and self.params['nopicks'] is True:
part = apParticle.getOneParticle(imgdata)
if not part:
apDisplay.printColor(
"\nrejecting unpicked image: " +
apDisplay.short(imgdata['filename']), "cyan")
imgassess = False
### ace stuff
if imgassess is not False:
imgassess = self.rejectAceInfo(imgdata)
### set global value
self.imgassess = imgassess
return
def _startLoop(self, imgdata):
"""
initilizes several parameters for a new image
and checks if it is okay to start processing image
"""
if self.params['parallel']:
if self.lockParallel(imgdata.dbid):
apDisplay.printMsg('%s locked by another parallel run in the rundir' % (apDisplay.shortenImageName(imgdata['filename'])))
return False
#calc images left
self.stats['imagesleft'] = self.stats['imagecount'] - self.stats['count']
#only if an image was processed last
if(self.stats['lastcount'] != self.stats['count']):
if self.params['background'] is False:
apDisplay.printColor( "\nStarting image %d ( skip:%d, remain:%d ) id:%d, file: %s"
%(self.stats['count'], self.stats['skipcount'], self.stats['imagesleft'],
imgdata.dbid, apDisplay.short(imgdata['filename']),),
"green")
elif self.stats['count'] % 80 == 0:
sys.stderr.write("\n")
self.stats['lastcount'] = self.stats['count']
if apDisplay.isDebugOn():
self._checkMemLeak()
# skip if image doesn't exist:
imgpath = os.path.join(imgdata['session']['image path'], imgdata['filename']+'.mrc')
if not os.path.isfile(imgpath):
apDisplay.printWarning(imgpath+" not found, skipping")
if self.params['parallel']:
self.unlockParallel(imgdata.dbid)
return False
# check to see if image has already been processed
if self._alreadyProcessed(imgdata):
if self.params['parallel']:
self.unlockParallel(imgdata.dbid)
return False
self.stats['waittime'] = 0
if self.reprocessImage(imgdata) is True:
if self.params['background'] is True:
sys.stderr.write(",")
else:
"""apDisplay.printMsg("reprocessing "+apDisplay.shortenImageName(imgdata['filename']))"""
else:
if self.params['background'] is True:
sys.stderr.write(".")
else:
"""apDisplay.printMsg("processing "+apDisplay.shortenImageName(imgdata['filename']))"""
return True
def validateAndInsertCTFData(imgdata, ctfvalues, rundata, rundir, fftpath=None, fftfreq=None):
"""
function to insert CTF values in database
"""
apDisplay.printMsg("Committing ctf parameters for "
+apDisplay.short(imgdata['filename'])+" to database")
if ctfvalues is None or not 'defocus2' in ctfvalues:
apDisplay.printWarning("No ctf values")
return False
### convert to common convention
ctfvalues = convertDefociToConvention(ctfvalues)
### check to make sure parameters are valid
isvalid = checkParams(ctfvalues)
if isvalid is False:
apDisplay.printWarning("Bad CTF values, insert but not create images")
### run the main CTF display program
opimagedir = os.path.join(rundir, "opimages")
if isvalid is True:
oldctfvalues = ctfvalues.copy()
ctfvalues = runCTFdisplayTools(imgdata, ctfvalues, opimagedir, fftpath, fftfreq)
# check if image creation failed
if ctfvalues is None:
ctfvalues = oldctfvalues
### clean rundir from all entries:
if not rundir.endswith("/"):
rundir += "/"
for key in ctfvalues.keys():
if isinstance(ctfvalues[key], str) and ctfvalues[key].startswith(rundir):
ctfvalues[key] = ctfvalues[key].replace(rundir, "")
### time to insert
ctfq = appiondata.ApCtfData()
ctfq['acerun'] = rundata
ctfq['image'] = imgdata
if debug is True:
apDisplay.printMsg("CTF data values")
print ctfvalues
for key in ctfq.keys():
if key in ctfvalues:
ctfq[key] = ctfvalues[key]
if debug is True:
apDisplay.printMsg("%s :: %s"%(key, ctfvalues[key]))
elif debug is True:
apDisplay.printMsg("SKIPPING %s :: %s"%(key, ctfvalues[key]))
ctfdb.printCtfData(ctfq)
ctfq.insert()
return
def getParticlePicks(self, imgdata, msg=True):
particles = []
if self.params['pickrunids'] is not None:
self.params['pickrunidlist'] = self.params['pickrunids'].split(",")
for pickrunid in self.params['pickrunidlist']:
#print pickrunid
newparticles = apParticle.getParticles(imgdata, pickrunid)
#apDisplay.printMsg("Found "+str(len(newparticles))+" particles for image "+apDisplay.short(imgdata['filename']))
particles.extend(newparticles)
targets = self.particlesToTargets(particles)
if msg is True:
apDisplay.printMsg("Found "+str(len(targets))+" particles for image "+apDisplay.short(imgdata['filename']))
return targets
def insertShift(imgdata,siblingdata,peak):
if not siblingdata or not peak:
apDisplay.printWarning("No sibling or peak found. No database insert")
return False
shiftq=appiondata.ApImageTransformationData()
shiftq['image1']=imgdata
shiftdata=shiftq.query()
if shiftdata:
apDisplay.printWarning("Shift values already in database")
return False
shiftq['image2']=siblingdata
shiftq['shiftx']=peak['shift'][1]
shiftq['shifty']=peak['shift'][0]
shiftq['scale']=peak['scalefactor']
if not peak['subfailed']:
shiftq['correlation']=peak['subpixel peak value']
else:
shiftq['correlation']=0.0
apDisplay.printMsg("Inserting shift beteween "+apDisplay.short(imgdata['filename'])+\
" and "+apDisplay.short(siblingdata['filename'])+" into database")
shiftq.insert()
return True
def insertShift(imgdata, siblingdata, peak):
if not siblingdata or not peak:
apDisplay.printWarning("No sibling or peak found. No database insert")
return False
shiftq = appiondata.ApImageTransformationData()
shiftq['image1'] = imgdata
shiftdata = shiftq.query()
if shiftdata:
apDisplay.printWarning("Shift values already in database")
return False
shiftq['image2'] = siblingdata
shiftq['shiftx'] = peak['shift'][1]
shiftq['shifty'] = peak['shift'][0]
shiftq['scale'] = peak['scalefactor']
if not peak['subfailed']:
shiftq['correlation'] = peak['subpixel peak value']
else:
shiftq['correlation'] = 0.0
apDisplay.printMsg("Inserting shift beteween "+apDisplay.short(imgdata['filename'])+\
" and "+apDisplay.short(siblingdata['filename'])+" into database")
shiftq.insert()
return True
def getBestTiltTransform(imgdata):
"""
For a given image get the best tilt transform
parameters in the database
"""
transdatas = []
## case 1: check if imgdata is image1
transq = appiondata.ApImageTiltTransformData()
transq['image1'] = imgdata
transdatas1 = transq.query()
if transdatas1:
transdatas.extend(transdatas1)
## case 2: check if imgdata is image2
transq = appiondata.ApImageTiltTransformData()
transq['image2'] = imgdata
transdatas2 = transq.query()
if transdatas2:
transdatas.extend(transdatas2)
### case 1: no transformation
if len(transdatas) == 0:
apDisplay.printWarning("no transformation found for image %s"
%(apDisplay.short(imgdata['filename'])))
return None
### case 2: single transformation, no sorting required
if len(transdatas) == 1:
apDisplay.printMsg("single transformation found with rmsd %.2f for image %s"
%(transdatas[0]['rmsd'], apDisplay.short(imgdata['filename'])))
return transdatas[0]
### case 3: more than one transformation, get one with lowest rmsd
besttransdata = transdatas[0]
bestrmsd = transdatas[0]['rmsd']
for transdata in transdatas:
if transdata['rmsd'] > 0 and transdata['rmsd'] < bestrmsd:
besttransdata = transdata
bestrmsd = transdata['rmsd']
apDisplay.printMsg("returning best transformation with rmsd %.2f for image %s"
%(bestrmsd, apDisplay.short(imgdata['filename'])))
return besttransdata
def findEMString(classavg, templatename, dwnimgname, ccmapfile, params):
#IMAGE INFO
if not os.path.isfile(dwnimgname):
apDisplay.printError("image file, " + dwnimgname + " was not found")
apDisplay.printMsg("image file, " + apDisplay.short(dwnimgname))
feed = dwnimgname + " "
#TEMPLATE INFO
if not os.path.isfile(templatename):
apDisplay.printError("template file, " + templatename +
" was not found")
apDisplay.printMsg("template file, " + templatename)
feed += templatename + " "
#CCMAPFILE INFO
apDisplay.printMsg("ccmap file, " + ccmapfile)
feed += ccmapfile + ' '
#BOXFILE OUTPUT
boxfilename = getBoxFileName(ccmapfile)
feed += boxfilename + ' '
#FAKE BOX SIZE
feed += '1 '
#PEAK_Theshold
feed += str(params["thresh"]) + " "
#CORRELATION MAP CALCULATION PARTICLE RADIUSa IN PIXELS
feed += str(int(params["diam"] /
(2.0 * params["apix"] * params["bin"]))) + " "
#PEAK SEARCH DISTANCE IN PIXELS
feed += str(
int(params["ccsearchmult"] * params["diam"] /
(params["apix"] * params["bin"]))) + " "
#ROTATION PARAMETERS
strt = str(params["startang" + str(classavg)])
end = str(params["endang" + str(classavg)])
incr = str(params["incrang" + str(classavg)])
feed += strt + ' ' + end + ' ' + incr + " "
#GRAPHICAL CARD DEVICE NUMBER
borderwidth = str(
int((params["diam"] / params["apix"] / params["bin"]) / 2) + 1)
feed += str(params["gcdev"]) + " "
return feed
def processImage(self, imgdata):
### reset global value
self.imgassess = True
### get initial assessment
imgassess = apDatabase.getImgAssessmentStatus(imgdata)
if imgassess is False:
return
if imgassess is None:
imgassess = True
### hidden image
imgview = apDatabase.getImgViewerStatus(imgdata)
if imgview is False:
apDisplay.printColor("\nrejecting hidden image: "+apDisplay.short(imgdata['filename']), "green")
self.imgassess = False
return
### tilt pair stuff
if imgassess is not False and self.params['notiltpairs'] is True:
imgassess = self.rejectTiltPairs(imgdata)
### picking stuff
if imgassess is not False and self.params['nopicks'] is True:
part = apParticle.getOneParticle(imgdata)
if not part:
apDisplay.printColor("\nrejecting unpicked image: "+apDisplay.short(imgdata['filename']), "cyan")
imgassess = False
### ace stuff
if imgassess is not False:
imgassess = self.rejectAceInfo(imgdata)
### set global value
self.imgassess = imgassess
return
def arrayToJpeg(numer, filename, normalize=True, msg=True, quality=85):
"""
takes a numpy and writes a JPEG
best for micrographs and photographs
"""
if normalize:
numer = imagenorm.maxNormalizeImage(numer)
else:
numer = numer*255
image = _arrayToImage(numer)
if msg is True:
apDisplay.printMsg("writing JPEG: "+apDisplay.short(filename))
image.save(filename, "JPEG", quality=quality)
return
def arrayToPng(numer, filename, normalize=True, msg=True):
"""
takes a numpy and writes a PNG
best for masks and line art
"""
if normalize:
numer = imagenorm.maxNormalizeImage(numer)
else:
numer = numer * 255
image = _arrayToImage(numer)
if msg is True:
apDisplay.printMsg("writing PNG: " + apDisplay.short(filename))
image.save(filename, "PNG")
return
def arrayToJpeg(numer, filename, normalize=True, msg=True, quality=85):
"""
takes a numpy and writes a JPEG
best for micrographs and photographs
"""
if normalize:
numer = imagenorm.maxNormalizeImage(numer)
else:
numer = numer * 255
image = _arrayToImage(numer)
if msg is True:
apDisplay.printMsg("writing JPEG: " + apDisplay.short(filename))
image.save(filename, "JPEG", quality=quality)
return
def arrayToPng(numer, filename, normalize=True, msg=True):
"""
takes a numpy and writes a PNG
best for masks and line art
"""
if normalize:
numer = imagenorm.maxNormalizeImage(numer)
else:
numer = numer*255
image = _arrayToImage(numer)
if msg is True:
apDisplay.printMsg("writing PNG: "+apDisplay.short(filename))
image.save(filename, "PNG")
return
def setImgViewerStatus(imgdata, status=None, msg=True):
"""
Function that sets the image status in the viewer
False: Image was hidden
True: Image is an exemplar
None: Image is visible
see 'ViewerImageStatus' table in dbemdata
"""
if status is False:
statusVal = 'hidden'
elif status is True:
statusVal = 'exemplar'
else:
print "skipping set viewer status"
return
currentstatus = getImgViewerStatus(imgdata)
if currentstatus is None:
#insert new
statusq = leginon.leginondata.ViewerImageStatus()
statusq['image'] = imgdata
statusq['status'] = statusVal
statusq.insert()
elif currentstatus != status:
#update column
dbconf = sinedon.getConfig('leginondata')
db = sinedon.sqldb.sqlDB(**dbconf)
q = ("UPDATE " + dbconf['db'] + ".`ViewerImageStatus` " +
"SET status = '" + statusVal +
("' WHERE `REF|AcquisitionImageData|image`=%d" %
(imgdata.dbid, )))
db.execute(q)
#check assessment
if msg is True:
finalassess = getImgViewerStatus(imgdata)
imgname = apDisplay.short(imgdata['filename'])
if finalassess is True:
astr = apDisplay.colorString("exemplar", "green")
elif finalassess is False:
astr = apDisplay.colorString("hidden", "red")
elif finalassess is None:
astr = apDisplay.colorString("none", "yellow")
apDisplay.printMsg("Final image assessment: " + astr + " (" + imgname +
")")
return
def setImgViewerStatus(imgdata, status=None, msg=True):
"""
Function that sets the image status in the viewer
False: Image was hidden
True: Image is an exemplar
None: Image is visible
see 'ViewerImageStatus' table in dbemdata
"""
if status is False:
statusVal = 'hidden'
elif status is True:
statusVal = 'exemplar'
else:
print "skipping set viewer status"
return
currentstatus = getImgViewerStatus(imgdata)
if currentstatus is None:
#insert new
statusq = leginon.leginondata.ViewerImageStatus()
statusq['image'] = imgdata
statusq['status'] = statusVal
statusq.insert()
elif currentstatus != status:
#update column
dbconf=sinedon.getConfig('leginondata')
db=sinedon.sqldb.sqlDB(**dbconf)
q= ( "UPDATE "+dbconf['db']+".`ViewerImageStatus` "
+"SET status = '"+statusVal
+ ("' WHERE `REF|AcquisitionImageData|image`=%d" % (imgdata.dbid,)))
db.execute(q)
#check assessment
if msg is True:
finalassess = getImgViewerStatus(imgdata)
imgname = apDisplay.short(imgdata['filename'])
if finalassess is True:
astr = apDisplay.colorString("exemplar", "green")
elif finalassess is False:
astr = apDisplay.colorString("hidden", "red")
elif finalassess is None:
astr = apDisplay.colorString("none", "yellow")
apDisplay.printMsg("Final image assessment: "+astr+" ("+imgname+")")
return
def checkPixelSize(self):
# make sure that images all have same pixel size:
# first get pixel size of first image:
self.params['apix'] = None
for imgdata in self.imgtree:
# get pixel size
imgname = imgdata['filename']
if imgname in self.donedict:
continue
if self.params['apix'] is None:
self.params['apix'] = apDatabase.getPixelSize(imgdata)
apDisplay.printMsg("Stack pixelsize = %.3f A"%(self.params['apix']))
if apDatabase.getPixelSize(imgdata) != self.params['apix']:
apDisplay.printMsg("Image pixelsize %.3f A != Stack pixelsize %.3f A"%(apDatabase.getPixelSize(imgdata), self.params['apix']))
apDisplay.printMsg("Problem image name: %s"%(apDisplay.short(imgdata['filename'])))
apDisplay.printError("This particle selection run contains images of varying pixelsizes, a stack cannot be created")
def findEMString(classavg, templatename, dwnimgname, ccmapfile, params):
#IMAGE INFO
if not os.path.isfile(dwnimgname):
apDisplay.printError("image file, "+dwnimgname+" was not found")
apDisplay.printMsg("image file, "+apDisplay.short(dwnimgname))
feed = dwnimgname+"\n"
#TEMPLATE INFO
if not os.path.isfile(templatename):
apDisplay.printError("template file, "+templatename+" was not found")
apDisplay.printMsg("template file, "+templatename)
feed += templatename + "\n"
#DUMMY VARIABLE; DOES NOTHING
feed += "-200.0\n"
#BINNED APIX
feed += str(params["apix"]*params["bin"])+"\n"
#PARTICLE DIAMETER
feed += str(params["diam"])+"\n"
#RUN ID FOR OUTPUT FILENAME
numstr = ("%03d" % classavg)+"\n"
#numstr = str(classavg%10)+"00\n"
feed += numstr
#ROTATION PARAMETERS
strt = str(params["startang"+str(classavg)])
end = str(params["endang"+str(classavg)])
incr = str(params["incrang"+str(classavg)])
feed += strt+','+end+','+incr+"\n"
# For FindEM2
# #MASK
# feed +="tmpmask.mrc\n"
# For FindEM2, comment out border
#BORDER WIDTH
borderwidth = str(int((params["diam"]/params["apix"]/params["bin"])/2)+1)
feed += borderwidth+"\n"
return feed
def findEMString(classavg, templatename, dwnimgname, ccmapfile, params):
#IMAGE INFO
if not os.path.isfile(dwnimgname):
apDisplay.printError("image file, "+dwnimgname+" was not found")
apDisplay.printMsg("image file, "+apDisplay.short(dwnimgname))
feed = dwnimgname+" "
#TEMPLATE INFO
if not os.path.isfile(templatename):
apDisplay.printError("template file, "+templatename+" was not found")
apDisplay.printMsg("template file, "+templatename)
feed += templatename + " "
#CCMAPFILE INFO
apDisplay.printMsg("ccmap file, "+ccmapfile)
feed += ccmapfile + ' '
#BOXFILE OUTPUT
boxfilename = getBoxFileName(ccmapfile)
feed += boxfilename + ' '
#FAKE BOX SIZE
feed += '1 '
#PEAK_Theshold
feed += str(params["thresh"])+" "
#CORRELATION MAP CALCULATION PARTICLE RADIUSa IN PIXELS
feed += str(int(params["diam"]/(2.0*params["apix"]*params["bin"])))+" "
#PEAK SEARCH DISTANCE IN PIXELS
feed += str(int(params["ccsearchmult"]*params["diam"]/(params["apix"]*params["bin"])))+" "
#ROTATION PARAMETERS
strt = str(params["startang"+str(classavg)])
end = str(params["endang"+str(classavg)])
incr = str(params["incrang"+str(classavg)])
feed += strt+' '+end+' '+incr+" "
#GRAPHICAL CARD DEVICE NUMBER
borderwidth = str(int((params["diam"]/params["apix"]/params["bin"])/2)+1)
feed += str(params["gcdev"])+" "
return feed
def arrayMaskToPngAlpha(numer,filename, msg=True):
"""
Create PNG file of a binary mask (array with only 0 and 1)
that uses the values in the alpha channel for transparency
"""
alpha=int(0.4*255)
numera = numer*alpha
numerones=numpy.ones(numpy.shape(numer))*255
imagedummy = _arrayToImage(numerones)
alphachannel = _arrayToImage(numera)
image = imagedummy.convert('RGBA')
image.putalpha(alphachannel)
if msg is True:
apDisplay.printMsg("writing alpha channel PNG mask: "+apDisplay.short(filename))
image.save(filename, "PNG")
return
def arrayMaskToPngAlpha(numer,filename, msg=True):
"""
Create PNG file of a binary mask (array with only 0 and 1)
that uses the values in the alpha channel for transparency
"""
alpha=int(0.4*255)
numera = numer*alpha
numerones=numpy.ones(numpy.shape(numer))*255
imagedummy = _arrayToImage(numerones)
alphachannel = _arrayToImage(numera)
image = imagedummy.convert('RGBA')
image.putalpha(alphachannel)
if msg is True:
apDisplay.printMsg("writing alpha channel PNG mask: "+apDisplay.short(filename))
image.save(filename, "PNG")
return
def getBestCtfValue(imgdata, sortType='res80', method=None, msg=True):
"""
takes an image and get the best ctfvalues for that image
"""
### get all ctf values
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfvalues = ctfq.query()
imgname = apDisplay.short(imgdata['filename'])
if msg is True:
print "Found %d ctf values"%(len(ctfvalues))
### check if it has values
if ctfvalues is None:
apDisplay.printWarning("no CTF values found in database for img %s"%(imgname))
return None
### find the best values
bestsortvalue = -1
bestctfvalue = None
for ctfvalue in ctfvalues:
if method is not None:
imgmethod = getCtfMethod(ctfvalue)
if method != imgmethod:
continue
sortvalue = getSortValueFromCtfQuery(ctfvalue, sortType)
if sortvalue is None:
continue
if msg is True:
print "%.3f -- %s"%(sortvalue, ctfvalue['acerun']['name'])
if sortvalue > bestsortvalue:
bestsortvalue = sortvalue
bestctfvalue = ctfvalue
if bestctfvalue is None:
apDisplay.printWarning("no best CTF value for image %s"%(imgname))
return None
if msg is True:
print "*** %.3f"%(bestsortvalue)
printCtfData(bestctfvalue)
return bestctfvalue
def insertImgAssessmentStatus(imgdata,
runname="run1",
assessment=None,
msg=True):
"""
Insert the assessment status
keep = True
reject = False
unassessed = None
"""
if assessment is True or assessment is False:
assessrun = appiondata.ApAssessmentRunData()
assessrun['session'] = imgdata['session']
#override to ALWAYS be 'run1'
#assessrun['name'] = runname
assessrun['name'] = "run1"
assessquery = appiondata.ApAssessmentData()
assessquery['image'] = imgdata
assessquery['assessmentrun'] = assessrun
assessquery['selectionkeep'] = assessment
assessquery.insert()
else:
apDisplay.printWarning("No image assessment made, invalid data: " +
str(assessment))
#check assessment
if msg is True:
finalassess = getImgAssessmentStatus(imgdata)
imgname = apDisplay.short(imgdata['filename'])
if finalassess is True:
astr = apDisplay.colorString("keep", "green")
elif finalassess is False:
astr = apDisplay.colorString("reject", "red")
elif finalassess is None:
astr = apDisplay.colorString("none", "yellow")
apDisplay.printMsg("Final image assessment: " + astr + " (" + imgname +
")")
return True
def _startLoop(self, info):
"""
appionLoop OVERRIDE
initilizes several parameters for a new image
and checks if it is okay to start processing image
"""
if info is None:
self.stats['lastimageskipped'] = True
self.stats['skipcount'] += 1
return False
name = info['filename']
# check to see if image of the same name is already in leginon
imgq = leginon.leginondata.AcquisitionImageData(session=self.session,
filename=name)
results = imgq.query(readimages=False)
if results:
apDisplay.printWarning("File %s.mrc exists at the destination" %
name)
apDisplay.printWarning("Skip Uploading")
self.stats['lastimageskipped'] = True
self.stats['skipcount'] += 1
return False
#calc images left
self.stats[
'imagesleft'] = self.stats['imagecount'] - self.stats['count']
#only if an image was processed last
if (self.stats['lastcount'] != self.stats['count']):
apDisplay.printColor( "\nStarting image "+str(self.stats['count'])\
+" ( skip:"+str(self.stats['skipcount'])+", remain:"\
+str(self.stats['imagesleft'])+" ) file: "\
+apDisplay.short(name), "green")
self.stats['lastcount'] = self.stats['count']
if apDisplay.isDebugOn():
self._checkMemLeak()
# check to see if image has already been processed
if self._alreadyProcessed(info):
return False
self.stats['waittime'] = 0
return True
def signatureString(templatename, dwnimgname, params):
# start batch mode
feed = " -c -film img -space picks"
#IMAGE INFO
if not os.path.isfile(dwnimgname):
apDisplay.printError("image file, " + dwnimgname + " was not found")
apDisplay.printMsg("image file, " + apDisplay.short(dwnimgname))
feed += " -source %s" % dwnimgname
#TEMPLATE INFO
if not os.path.isfile(templatename):
apDisplay.printError("template file, " + templatename +
" was not found")
apDisplay.printMsg("template file, " + templatename)
feed += " -template %s" % templatename
#BINNED APIX
feed += " -pixelsize %.3f" % (params["apix"] * params["bin"])
#PARTICLE DIAMETER
feed += " -partsize %i" % int(params['diam'])
#BORDER WIDTH
borderwidth = int(
(params["diam"] / params["apix"] / params["bin"]) / 2) + 1
feed += " -margin %i" % borderwidth
#threshold
feed += " -lcf-thresh %.2f" % params['thresh']
#particle distance
feed += " -partdist %i" % (params['overlapmult'] * params['diam'])
#resizing set to 4 always (doesn't work otherwise)
feed += " -resize 4"
return feed
def start(self):
imgfilename2peaklist = self.readFileToPeakTree()
imglist = imgfilename2peaklist.keys()
apDisplay.printMsg("Getting image data from database")
imgtree = apDatabase.getSpecificImagesFromDB(imglist, self.sessiondata)
if imgtree[0]['session']['name'] != self.sessiondata['name']:
apDisplay.printError("Session and Image do not match "+imgtree[0]['filename'])
### insert params for manual picking
rundata = self.insertManualParams()
# upload Particles
for imgdata in imgtree:
### check session
if imgdata['session']['name'] != self.sessiondata['name']:
apDisplay.printError("Session and Image do not match "+imgdata['filename'])
peaktree = imgfilename2peaklist[imgdata['filename']]
apDisplay.printMsg("%d particles for image %s"
%(len(peaktree), apDisplay.short(imgdata['filename'])))
if self.params['commit'] is True:
apParticle.insertParticlePeaks(peaktree, imgdata, self.params['runname'], msg=True)