def getCtfValueForCtfRunId(imgdata, ctfrunid=None, msg=False):
"""
takes an image and get the ctf value for that image for the specified ctf estimation run
specified methods can be: ace2 or ctffind
"""
if ctfrunid is None:
apDisplay.printError("This function requires a ctf_estimation_runid")
ctfrundata = appiondata.ApAceRunData.direct_query(ctfrunid)
if ctfrundata is None:
apDisplay.printError("ctf_estimation_runid not found")
### get all ctf values
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfq['acerun'] = ctfrundata
ctfvalues = ctfq.query()
### check if it has values
if ctfvalues is None:
return None
### find the value
ctfdata = None
if len(ctfvalues) == 1:
if msg is True:
printCtfData(ctfvalues[0])
return ctfvalues[0]
elif len(ctfvalues) > 1:
apDisplay.printWarning("more than one run found for ctfrunid %d and image %s"
%(ctfrunid, apDisplay.short(imgdata['filename'])))
return ctfvalues[-1]
return None
def validateAndInsertCTFData(imgdata,
ctfvalues,
rundata,
rundir,
fftpath=None,
fftfreq=None):
"""
function to insert CTF values in database
"""
apDisplay.printMsg("Committing ctf parameters for " +
apDisplay.short(imgdata['filename']) + " to database")
if ctfvalues is None or not 'defocus2' in ctfvalues:
apDisplay.printWarning("No ctf values")
return False
### convert to common convention
ctfvalues = convertDefociToConvention(ctfvalues)
### check to make sure parameters are valid
isvalid = checkParams(ctfvalues)
if isvalid is False:
apDisplay.printWarning("Bad CTF values, insert but not create images")
### run the main CTF display program
opimagedir = os.path.join(rundir, "opimages")
if isvalid is True:
oldctfvalues = ctfvalues.copy()
ctfvalues = runCTFdisplayTools(imgdata, ctfvalues, opimagedir, fftpath,
fftfreq)
# check if image creation failed
if ctfvalues is None:
ctfvalues = oldctfvalues
### clean rundir from all entries:
if not rundir.endswith("/"):
rundir += "/"
for key in ctfvalues.keys():
if isinstance(ctfvalues[key],
str) and ctfvalues[key].startswith(rundir):
ctfvalues[key] = ctfvalues[key].replace(rundir, "")
### time to insert
ctfq = appiondata.ApCtfData()
ctfq['acerun'] = rundata
ctfq['image'] = imgdata
if debug is True:
apDisplay.printMsg("CTF data values")
print ctfvalues
for key in ctfq.keys():
if key in ctfvalues:
ctfq[key] = ctfvalues[key]
if debug is True:
apDisplay.printMsg("%s :: %s" % (key, ctfvalues[key]))
elif debug is True:
apDisplay.printMsg("SKIPPING %s :: %s" % (key, ctfvalues[key]))
ctfdb.printCtfData(ctfq)
ctfq.insert()
return
def getBestTiltCtfValueForImage(imgdata):
"""
takes an image and get the tilted ctf parameters for that image
"""
#print "ERROR getBestTiltCtfValueForImage(), use getBestCtfByResolution() instead"
#sys.exit(1)
### get all ctf values
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfvalues = ctfq.query()
bestctftiltvalue = None
cross_correlation = 0.0
for ctfvalue in ctfvalues:
if ctfvalue['acerun'] is not None:
if bestctftiltvalue is None:
cross_correlation = ctfvalue['cross_correlation']
bestctftiltvalue = ctfvalue
else:
if cross_correlation < ctfvalue['cross_correlation']:
cross_correlation = ctfvalue['cross_correlation']
bestctftiltvalue = ctfvalue
return bestctftiltvalue
def upgradeAppionDB(self):
if not self.appion_dbupgrade.tableExists('ApCtfData'):
return
# CtfData
r = appiondata.ApCtfData().query()
r.reverse()
for data in r:
if data['angle_astigmatism']:
# Only change non-zero or non- None results
q = "UPDATE `ApCtfData` SET `angle_astigmatism` = %e WHERE `DEF_id` =%d" % (
-data['angle_astigmatism'], data.dbid)
self.appion_dbupgrade.executeCustomSQL(q)
def getCtfValueForImage(imgdata,
ctf_estimation_runid=None,
ctfavg=True,
msg=True,
method=None):
"""
takes an image and get the ctf value for that image for the specified ctf estimation run
specified methods can be: ace2 or ctffind
"""
#print "ERROR getCtfValueForImage(), use getBestCtfByResolution() instead"
#sys.exit(1)
if ctf_estimation_runid is None:
apDisplay.printError("This function requires a ctf_estimation_runid")
### get all ctf values
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfvalues = ctfq.query()
### check if it has values
if ctfvalues is None:
return None, None
### find the value
ctfdata = None
for ctfvalue in ctfvalues:
### limit to specific method if requested:
if method is not None and getCtfMethod(ctfvalue) != method:
continue
### specify ID for CTF run
if ctfvalue['acerun'].dbid == ctf_estimation_runid:
### make sure that CTF values were estimated
if ctfvalue['defocus1'] is None and ctfvalue['defocus2'] is None:
apDisplay.printWarning(
"CTF estimation using the specified parameters did not work"
)
return None, None
else:
ctfdata = ctfvalue
break
# no data found with the criteria
if ctfdata is None:
return None, None
conf = calculateConfidenceScore(ctfvalue, ctfavg)
if msg is True:
printCtfData(ctfvalue)
return ctfvalue, conf
def getBestCtfByResolution(imgdata, msg=True, method=None):
"""
takes an image and get the best ctfvalues for that image
specified methods can be: ace2 or ctffind
"""
### get all ctf values
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfvalues = ctfq.query()
if msg is True:
print "Found %d ctf values" % (len(ctfvalues))
### check if it has values
if ctfvalues is None:
print "no CTF values found in database"
return None
### find the best values
bestres50 = 1000.0
bestctfvalue = None
for ctfvalue in ctfvalues:
conf = calculateConfidenceScore(ctfvalue)
if conf < 0.3:
continue
if method is not None:
imgmethod = getCtfMethod(ctfvalue)
if method != imgmethod:
continue
res50 = ctfvalue['resolution_50_percent']
if res50 is None:
continue
if msg is True:
print "%.3f -- %s" % (res50, ctfvalue['acerun']['name'])
if res50 is None:
continue
if res50 < bestres50:
bestres50 = res50
bestctfvalue = ctfvalue
if bestctfvalue is None:
print "no best value"
return None
if msg is True:
print "*** %.3f" % (bestres50)
printCtfData(bestctfvalue)
return bestctfvalue
def hasRecentEntry(self):
if self.appion_dbupgrade.tableExists('ApCtfData'):
results = appiondata.ApCtfData().query(results=1)
if results:
ctfdata = results[0]
if ctfdata.timestamp > self.checktime:
return True
if self.appion_dbupgrade.tableExists('ApStackRunData'):
results = appiondata.ApStackRunData().query(results=1)
if results:
stackrundata = results[0]
if stackrundata.timestamp > self.checktime and stackrundata[
'stackParams']['phaseFlipped']:
return True
return False
def getBestCtfValueForImage(imgdata, ctfavg=True, msg=True, method=None):
"""
takes an image and get the best ctfvalues for that image
specified methods can be: ace2 or ctffind
"""
#print "ERROR getBestCtfValueForImage(), use getBestCtfByResolution() instead"
#sys.exit(1)
### get all ctf values
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfvalues = ctfq.query()
### check if it has values
if ctfvalues is None:
return None, None
### find the best values
bestconf = 0.0
bestctfvalue = None
count = 0
for ctfvalue in ctfvalues:
count += 1
### limit to specific method if requested:
if method is not None and getCtfMethod(ctfvalue) != method:
continue
conf = calculateConfidenceScore(ctfvalue, ctfavg)
if msg is True:
if ctfvalue['resolution_50_percent']:
print "%d conf %.4f (res %.1f) (?? %.4f)" % (
count, conf, ctfvalue['resolution_50_percent'], bestconf)
else:
print "%d conf %.4f (?? %.4f)" % (count, conf, bestconf)
if conf > bestconf:
bestconf = conf
bestctfvalue = ctfvalue
if bestctfvalue == None:
return None, None
if msg is True:
print "selected conf %.2f" % (bestconf)
printCtfData(bestctfvalue)
return bestctfvalue, bestconf
def scanAppionDB(self):
if self.appion_dbtools.tableExists('ApCtfData'):
results = appiondata.ApCtfData().query(results=1)
if results:
ctfdata = results[0]
if ctfdata.timestamp > self.checktime:
print "\033[35m%s has new ApCtfData in %d days %d hours\033[0m" % (self.appion_dbtools.getDatabaseName(),-self.deltadays,-self.deltahours)
if self.appion_dbtools.tableExists('ApStackRunData'):
results = appiondata.ApStackRunData().query(results=1)
if results:
stackrundata = results[0]
if stackrundata['stackParams']['phaseFlipped']:
stackpartr = appiondata.ApStackParticleData(stackRun=stackrundata).query(results=1)
if stackpartr:
stackpartdata = stackpartr[0]
if stackpartdata.timestamp > self.checktime:
print "\033[35m%s has new particle inserted to Stack with phase flip in %d days %d hours\033[0m" % (self.appion_dbtools.getDatabaseName(),-self.deltadays,-self.deltahours)
def getBestCtfValue(imgdata, sortType='res80', method=None, msg=True):
"""
takes an image and get the best ctfvalues for that image
"""
### get all ctf values
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfvalues = ctfq.query()
imgname = apDisplay.short(imgdata['filename'])
if msg is True:
print "Found %d ctf values"%(len(ctfvalues))
### check if it has values
if ctfvalues is None:
apDisplay.printWarning("no CTF values found in database for img %s"%(imgname))
return None
### find the best values
bestsortvalue = -1
bestctfvalue = None
for ctfvalue in ctfvalues:
if method is not None:
imgmethod = getCtfMethod(ctfvalue)
if method != imgmethod:
continue
sortvalue = getSortValueFromCtfQuery(ctfvalue, sortType)
if sortvalue is None:
continue
if msg is True:
print "%.3f -- %s"%(sortvalue, ctfvalue['acerun']['name'])
if sortvalue > bestsortvalue:
bestsortvalue = sortvalue
bestctfvalue = ctfvalue
if bestctfvalue is None:
apDisplay.printWarning("no best CTF value for image %s"%(imgname))
return None
if msg is True:
print "*** %.3f"%(bestsortvalue)
printCtfData(bestctfvalue)
return bestctfvalue
def printCtfSummary(params, imgtree):
"""
prints a histogram of the best ctfvalues for the session
"""
# if there are no images in the imgtree, there was no new processing done, so exit this function early.
if not imgtree:
apDisplay.printWarning("There are no new results to summarize.")
return
sys.stderr.write("processing CTF histogram...\n")
### get best ctf values for each image
ctfhistconf = []
ctfhistval = []
for imgdata in imgtree:
if params[
'norejects'] is True and apDatabase.getSiblingImgAssessmentStatus(
imgdata) is False:
continue
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfvalues = ctfq.query()
### check if it has values
if ctfvalues is None:
continue
### find the best values
bestconf = 0.0
bestctfvalue = None
for ctfvalue in ctfvalues:
conf = calculateConfidenceScore(ctfvalue, False)
if conf > bestconf:
bestconf = conf
bestctfvalue = ctfvalue
ctfhistconf.append(bestconf)
ctfhistval.append(bestctfvalue)
ctfhistconf.sort()
confhist = {}
yspan = 20.0
minconf = ctfhistconf[0]
maxconf = ctfhistconf[len(ctfhistconf) - 1]
maxcount = 0
for conf in ctfhistconf:
c2 = round(conf * yspan, 0) / float(yspan)
if c2 in confhist:
confhist[c2] += 1
if confhist[c2] > maxcount:
maxcount = confhist[c2]
else:
confhist[c2] = 1
if maxcount > 70:
scale = 70.0 / float(maxcount)
sys.stderr.write(" * = " + str(round(scale, 1)) + " images\n")
else:
scale = 1.0
colorstr = {}
for i in range(int(yspan + 1)):
j = float(i) / yspan
if j < 0.5:
colorstr[j] = "red"
elif j < 0.8:
colorstr[j] = "yellow"
else:
colorstr[j] = "green"
sys.stderr.write("Confidence histogram:\n")
for i in range(int(yspan + 1)):
j = float(i) / yspan
if j < minconf - 1.0 / yspan:
continue
jstr = "%1.2f" % j
jstr = apDisplay.rightPadString(jstr, 5)
sys.stderr.write(jstr + "> ")
if j in confhist:
for k in range(int(confhist[j] * scale)):
sys.stderr.write(apDisplay.color("*", colorstr[j]))
sys.stderr.write("\n")
