def getCtfValueForCtfRunId(imgdata, ctfrunid=None, msg=False): """ takes an image and get the ctf value for that image for the specified ctf estimation run specified methods can be: ace2 or ctffind """ if ctfrunid is None: apDisplay.printError("This function requires a ctf_estimation_runid") ctfrundata = appiondata.ApAceRunData.direct_query(ctfrunid) if ctfrundata is None: apDisplay.printError("ctf_estimation_runid not found") ### get all ctf values ctfq = appiondata.ApCtfData() ctfq['image'] = imgdata ctfq['acerun'] = ctfrundata ctfvalues = ctfq.query() ### check if it has values if ctfvalues is None: return None ### find the value ctfdata = None if len(ctfvalues) == 1: if msg is True: printCtfData(ctfvalues[0]) return ctfvalues[0] elif len(ctfvalues) > 1: apDisplay.printWarning("more than one run found for ctfrunid %d and image %s" %(ctfrunid, apDisplay.short(imgdata['filename']))) return ctfvalues[-1] return None
def validateAndInsertCTFData(imgdata, ctfvalues, rundata, rundir, fftpath=None, fftfreq=None): """ function to insert CTF values in database """ apDisplay.printMsg("Committing ctf parameters for " + apDisplay.short(imgdata['filename']) + " to database") if ctfvalues is None or not 'defocus2' in ctfvalues: apDisplay.printWarning("No ctf values") return False ### convert to common convention ctfvalues = convertDefociToConvention(ctfvalues) ### check to make sure parameters are valid isvalid = checkParams(ctfvalues) if isvalid is False: apDisplay.printWarning("Bad CTF values, insert but not create images") ### run the main CTF display program opimagedir = os.path.join(rundir, "opimages") if isvalid is True: oldctfvalues = ctfvalues.copy() ctfvalues = runCTFdisplayTools(imgdata, ctfvalues, opimagedir, fftpath, fftfreq) # check if image creation failed if ctfvalues is None: ctfvalues = oldctfvalues ### clean rundir from all entries: if not rundir.endswith("/"): rundir += "/" for key in ctfvalues.keys(): if isinstance(ctfvalues[key], str) and ctfvalues[key].startswith(rundir): ctfvalues[key] = ctfvalues[key].replace(rundir, "") ### time to insert ctfq = appiondata.ApCtfData() ctfq['acerun'] = rundata ctfq['image'] = imgdata if debug is True: apDisplay.printMsg("CTF data values") print ctfvalues for key in ctfq.keys(): if key in ctfvalues: ctfq[key] = ctfvalues[key] if debug is True: apDisplay.printMsg("%s :: %s" % (key, ctfvalues[key])) elif debug is True: apDisplay.printMsg("SKIPPING %s :: %s" % (key, ctfvalues[key])) ctfdb.printCtfData(ctfq) ctfq.insert() return
def getBestTiltCtfValueForImage(imgdata): """ takes an image and get the tilted ctf parameters for that image """ #print "ERROR getBestTiltCtfValueForImage(), use getBestCtfByResolution() instead" #sys.exit(1) ### get all ctf values ctfq = appiondata.ApCtfData() ctfq['image'] = imgdata ctfvalues = ctfq.query() bestctftiltvalue = None cross_correlation = 0.0 for ctfvalue in ctfvalues: if ctfvalue['acerun'] is not None: if bestctftiltvalue is None: cross_correlation = ctfvalue['cross_correlation'] bestctftiltvalue = ctfvalue else: if cross_correlation < ctfvalue['cross_correlation']: cross_correlation = ctfvalue['cross_correlation'] bestctftiltvalue = ctfvalue return bestctftiltvalue
def upgradeAppionDB(self): if not self.appion_dbupgrade.tableExists('ApCtfData'): return # CtfData r = appiondata.ApCtfData().query() r.reverse() for data in r: if data['angle_astigmatism']: # Only change non-zero or non- None results q = "UPDATE `ApCtfData` SET `angle_astigmatism` = %e WHERE `DEF_id` =%d" % ( -data['angle_astigmatism'], data.dbid) self.appion_dbupgrade.executeCustomSQL(q)
def getCtfValueForImage(imgdata, ctf_estimation_runid=None, ctfavg=True, msg=True, method=None): """ takes an image and get the ctf value for that image for the specified ctf estimation run specified methods can be: ace2 or ctffind """ #print "ERROR getCtfValueForImage(), use getBestCtfByResolution() instead" #sys.exit(1) if ctf_estimation_runid is None: apDisplay.printError("This function requires a ctf_estimation_runid") ### get all ctf values ctfq = appiondata.ApCtfData() ctfq['image'] = imgdata ctfvalues = ctfq.query() ### check if it has values if ctfvalues is None: return None, None ### find the value ctfdata = None for ctfvalue in ctfvalues: ### limit to specific method if requested: if method is not None and getCtfMethod(ctfvalue) != method: continue ### specify ID for CTF run if ctfvalue['acerun'].dbid == ctf_estimation_runid: ### make sure that CTF values were estimated if ctfvalue['defocus1'] is None and ctfvalue['defocus2'] is None: apDisplay.printWarning( "CTF estimation using the specified parameters did not work" ) return None, None else: ctfdata = ctfvalue break # no data found with the criteria if ctfdata is None: return None, None conf = calculateConfidenceScore(ctfvalue, ctfavg) if msg is True: printCtfData(ctfvalue) return ctfvalue, conf
def getBestCtfByResolution(imgdata, msg=True, method=None): """ takes an image and get the best ctfvalues for that image specified methods can be: ace2 or ctffind """ ### get all ctf values ctfq = appiondata.ApCtfData() ctfq['image'] = imgdata ctfvalues = ctfq.query() if msg is True: print "Found %d ctf values" % (len(ctfvalues)) ### check if it has values if ctfvalues is None: print "no CTF values found in database" return None ### find the best values bestres50 = 1000.0 bestctfvalue = None for ctfvalue in ctfvalues: conf = calculateConfidenceScore(ctfvalue) if conf < 0.3: continue if method is not None: imgmethod = getCtfMethod(ctfvalue) if method != imgmethod: continue res50 = ctfvalue['resolution_50_percent'] if res50 is None: continue if msg is True: print "%.3f -- %s" % (res50, ctfvalue['acerun']['name']) if res50 is None: continue if res50 < bestres50: bestres50 = res50 bestctfvalue = ctfvalue if bestctfvalue is None: print "no best value" return None if msg is True: print "*** %.3f" % (bestres50) printCtfData(bestctfvalue) return bestctfvalue
def hasRecentEntry(self): if self.appion_dbupgrade.tableExists('ApCtfData'): results = appiondata.ApCtfData().query(results=1) if results: ctfdata = results[0] if ctfdata.timestamp > self.checktime: return True if self.appion_dbupgrade.tableExists('ApStackRunData'): results = appiondata.ApStackRunData().query(results=1) if results: stackrundata = results[0] if stackrundata.timestamp > self.checktime and stackrundata[ 'stackParams']['phaseFlipped']: return True return False
def getBestCtfValueForImage(imgdata, ctfavg=True, msg=True, method=None): """ takes an image and get the best ctfvalues for that image specified methods can be: ace2 or ctffind """ #print "ERROR getBestCtfValueForImage(), use getBestCtfByResolution() instead" #sys.exit(1) ### get all ctf values ctfq = appiondata.ApCtfData() ctfq['image'] = imgdata ctfvalues = ctfq.query() ### check if it has values if ctfvalues is None: return None, None ### find the best values bestconf = 0.0 bestctfvalue = None count = 0 for ctfvalue in ctfvalues: count += 1 ### limit to specific method if requested: if method is not None and getCtfMethod(ctfvalue) != method: continue conf = calculateConfidenceScore(ctfvalue, ctfavg) if msg is True: if ctfvalue['resolution_50_percent']: print "%d conf %.4f (res %.1f) (?? %.4f)" % ( count, conf, ctfvalue['resolution_50_percent'], bestconf) else: print "%d conf %.4f (?? %.4f)" % (count, conf, bestconf) if conf > bestconf: bestconf = conf bestctfvalue = ctfvalue if bestctfvalue == None: return None, None if msg is True: print "selected conf %.2f" % (bestconf) printCtfData(bestctfvalue) return bestctfvalue, bestconf
def scanAppionDB(self): if self.appion_dbtools.tableExists('ApCtfData'): results = appiondata.ApCtfData().query(results=1) if results: ctfdata = results[0] if ctfdata.timestamp > self.checktime: print "\033[35m%s has new ApCtfData in %d days %d hours\033[0m" % (self.appion_dbtools.getDatabaseName(),-self.deltadays,-self.deltahours) if self.appion_dbtools.tableExists('ApStackRunData'): results = appiondata.ApStackRunData().query(results=1) if results: stackrundata = results[0] if stackrundata['stackParams']['phaseFlipped']: stackpartr = appiondata.ApStackParticleData(stackRun=stackrundata).query(results=1) if stackpartr: stackpartdata = stackpartr[0] if stackpartdata.timestamp > self.checktime: print "\033[35m%s has new particle inserted to Stack with phase flip in %d days %d hours\033[0m" % (self.appion_dbtools.getDatabaseName(),-self.deltadays,-self.deltahours)
def getBestCtfValue(imgdata, sortType='res80', method=None, msg=True): """ takes an image and get the best ctfvalues for that image """ ### get all ctf values ctfq = appiondata.ApCtfData() ctfq['image'] = imgdata ctfvalues = ctfq.query() imgname = apDisplay.short(imgdata['filename']) if msg is True: print "Found %d ctf values"%(len(ctfvalues)) ### check if it has values if ctfvalues is None: apDisplay.printWarning("no CTF values found in database for img %s"%(imgname)) return None ### find the best values bestsortvalue = -1 bestctfvalue = None for ctfvalue in ctfvalues: if method is not None: imgmethod = getCtfMethod(ctfvalue) if method != imgmethod: continue sortvalue = getSortValueFromCtfQuery(ctfvalue, sortType) if sortvalue is None: continue if msg is True: print "%.3f -- %s"%(sortvalue, ctfvalue['acerun']['name']) if sortvalue > bestsortvalue: bestsortvalue = sortvalue bestctfvalue = ctfvalue if bestctfvalue is None: apDisplay.printWarning("no best CTF value for image %s"%(imgname)) return None if msg is True: print "*** %.3f"%(bestsortvalue) printCtfData(bestctfvalue) return bestctfvalue
def printCtfSummary(params, imgtree): """ prints a histogram of the best ctfvalues for the session """ # if there are no images in the imgtree, there was no new processing done, so exit this function early. if not imgtree: apDisplay.printWarning("There are no new results to summarize.") return sys.stderr.write("processing CTF histogram...\n") ### get best ctf values for each image ctfhistconf = [] ctfhistval = [] for imgdata in imgtree: if params[ 'norejects'] is True and apDatabase.getSiblingImgAssessmentStatus( imgdata) is False: continue ctfq = appiondata.ApCtfData() ctfq['image'] = imgdata ctfvalues = ctfq.query() ### check if it has values if ctfvalues is None: continue ### find the best values bestconf = 0.0 bestctfvalue = None for ctfvalue in ctfvalues: conf = calculateConfidenceScore(ctfvalue, False) if conf > bestconf: bestconf = conf bestctfvalue = ctfvalue ctfhistconf.append(bestconf) ctfhistval.append(bestctfvalue) ctfhistconf.sort() confhist = {} yspan = 20.0 minconf = ctfhistconf[0] maxconf = ctfhistconf[len(ctfhistconf) - 1] maxcount = 0 for conf in ctfhistconf: c2 = round(conf * yspan, 0) / float(yspan) if c2 in confhist: confhist[c2] += 1 if confhist[c2] > maxcount: maxcount = confhist[c2] else: confhist[c2] = 1 if maxcount > 70: scale = 70.0 / float(maxcount) sys.stderr.write(" * = " + str(round(scale, 1)) + " images\n") else: scale = 1.0 colorstr = {} for i in range(int(yspan + 1)): j = float(i) / yspan if j < 0.5: colorstr[j] = "red" elif j < 0.8: colorstr[j] = "yellow" else: colorstr[j] = "green" sys.stderr.write("Confidence histogram:\n") for i in range(int(yspan + 1)): j = float(i) / yspan if j < minconf - 1.0 / yspan: continue jstr = "%1.2f" % j jstr = apDisplay.rightPadString(jstr, 5) sys.stderr.write(jstr + "> ") if j in confhist: for k in range(int(confhist[j] * scale)): sys.stderr.write(apDisplay.color("*", colorstr[j])) sys.stderr.write("\n")