def printHelp(xmldict, exit=True):
"""
print out help info for a function with XML file
"""
paramlist = xmldict.keys()
paramlist.sort()
maxlen = 0
maxlentype = 0
for param in paramlist:
if len(param) > maxlen:
maxlen = len(param)
if 'type' in xmldict[param] and len(
xmldict[param]['type']) > maxlentype:
maxlentype = len(xmldict[param]['type'])
for param in paramlist:
if not 'alias' in xmldict[param] and \
(not 'modify' in xmldict[param] or str2bool(xmldict[param]['modify']) == True):
outstr = " "
outstr += apDisplay.color(apDisplay.rightPadString(param, maxlen),
"green")
outstr += " :"
if 'type' in xmldict[param] and xmldict[param]['type'] != None:
outstr += " (" + apDisplay.rightPadString(
xmldict[param]['type'] + ")", maxlentype + 1)
outstr += " :"
if 'required' in xmldict[param] and str2bool(
xmldict[param]['required']) == True:
outstr += apDisplay.color(" REQ", "red")
if 'description' in xmldict[
param] and xmldict[param]['description'] != None:
outstr += " " + xmldict[param]['description']
elif 'name' in xmldict[param] and xmldict[param]['name'] != None:
outstr += " " + xmldict[param]['name']
if 'default' in xmldict[
param] and xmldict[param]['default'] != None:
if 'nargs' in xmldict[param] and xmldict[param][
'nargs'] is not None and xmldict[param]['nargs'] > 1:
defstr = " (default: "
for i in range(len(xmldict[param]['default'])):
defstr += str(xmldict[param]['default'][i]) + ","
defstr = defstr[:-1] + ")"
outstr += apDisplay.color(defstr, "cyan")
else:
outstr += apDisplay.color(
" (default: " + str(xmldict[param]['default']) + ")",
"cyan")
if 'example' in xmldict[
param] and xmldict[param]['example'] != None:
outstr += " (example: " + str(xmldict[param]['example']) + ")"
print outstr
if exit is True:
sys.exit(1)
def printSymmetries():
symq = appiondata.ApSymmetryData()
syms = symq.query()
sys.stderr.write("ID NAME DESCRIPTION\n")
sys.stderr.write("-- ---- -----------\n")
syms.sort(compSymm)
for s in syms:
name = s['eman_name']
name = re.sub('Icosahedral', 'Icos', name)
sys.stderr.write(
apDisplay.colorString(apDisplay.rightPadString(s.dbid, 3), "green")
+ " " + apDisplay.rightPadString(name, 5) + " " +
apDisplay.rightPadString(s['description'], 60) + "\n")
def printSymmetries():
symq = appiondata.ApSymmetryData()
syms = symq.query()
sys.stderr.write("ID NAME DESCRIPTION\n")
sys.stderr.write("-- ---- -----------\n")
syms.sort(compSymm)
for s in syms:
name = s['eman_name']
name = re.sub('Icosahedral', 'Icos', name)
sys.stderr.write(
apDisplay.colorString(apDisplay.rightPadString(s.dbid,3),"green")+" "
+apDisplay.rightPadString(name,5)+" "
+apDisplay.rightPadString(s['description'],60)+"\n"
)
def printHelp(xmldict, exit=True):
"""
print out help info for a function with XML file
"""
paramlist = xmldict.keys()
paramlist.sort()
maxlen = 0
maxlentype = 0
for param in paramlist:
if len(param) > maxlen:
maxlen = len(param)
if 'type' in xmldict[param] and len(xmldict[param]['type']) > maxlentype:
maxlentype = len(xmldict[param]['type'])
for param in paramlist:
if not 'alias' in xmldict[param] and \
(not 'modify' in xmldict[param] or str2bool(xmldict[param]['modify']) == True):
outstr = " "
outstr += apDisplay.color(apDisplay.rightPadString(param,maxlen),"green")
outstr += " :"
if 'type' in xmldict[param] and xmldict[param]['type'] != None:
outstr += " ("+apDisplay.rightPadString(xmldict[param]['type']+")",maxlentype+1)
outstr += " :"
if 'required' in xmldict[param] and str2bool(xmldict[param]['required']) == True:
outstr += apDisplay.color(" REQ","red")
if 'description' in xmldict[param] and xmldict[param]['description'] != None:
outstr += " "+xmldict[param]['description']
elif 'name' in xmldict[param] and xmldict[param]['name'] != None:
outstr += " "+xmldict[param]['name']
if 'default' in xmldict[param] and xmldict[param]['default'] != None:
if 'nargs' in xmldict[param] and xmldict[param]['nargs'] is not None and xmldict[param]['nargs'] > 1:
defstr = " (default: "
for i in range(len(xmldict[param]['default'])):
defstr += str(xmldict[param]['default'][i])+","
defstr = defstr[:-1]+")"
outstr += apDisplay.color(defstr,"cyan")
else:
outstr += apDisplay.color(" (default: "+str(xmldict[param]['default'])+")","cyan")
if 'example' in xmldict[param] and xmldict[param]['example'] != None:
outstr += " (example: "+str(xmldict[param]['example'])+")"
print outstr
if exit is True:
sys.exit(1)
def fancyPrintDict(pdict): """ prints out two levels of a dictionary """ pkeys = pdict.keys() pkeys.sort() maxlen = 0 print "----------" for p in pkeys: if len(p) > maxlen: maxlen = len(p) for p in pkeys: print " ",apDisplay.rightPadString(p+":",maxlen+2),\ apDisplay.colorType(pdict[p]) print "----------"
def fancyPrintDict(pdict):
"""
prints out two levels of a dictionary
"""
pkeys = pdict.keys()
pkeys.sort()
maxlen = 0
print "----------"
for p in pkeys:
if len(p) > maxlen: maxlen = len(p)
for p in pkeys:
print " ",apDisplay.rightPadString(p+":",maxlen+2),\
apDisplay.colorType(pdict[p])
print "----------"
def modelCTFNoise(self, xdata, ctfdata, contraint="below"):
"""
Master control function to fit the CTF noise function
xdata - should be in inverse Angstroms
"""
t0 = time.time()
### need to reduce precision of the xdata
### otherwise it takes too long, with no better of a fit
xdata = xdata.astype(numpy.float32)
if self.debug is True:
apDisplay.printColor("CTF limits %.1f A -->> %.1fA"
%(1./xdata.min(), 1./xdata.max()), "cyan")
if contraint == "above":
if self.debug is True:
print "constrained above function"
contraintFunction = self.modelConstFunAbove
#filterctfdata = scipy.ndimage.maximum_filter(ctfdata, size=2)
#for i in range(1):
# filterctfdata = (filterctfdata + scipy.ndimage.maximum_filter(filterctfdata, size=2))/2.0
#firstmax = filterctfdata[0:250].max()
#filterctfdata = numpy.where(filterctfdata>firstmax, firstmax, filterctfdata)
#filterctfdata = self.upwardLeftMonotonicFilter(ctfdata)
filterctfdata = ctfdata
else:
if self.debug is True:
print "constrained below function"
contraintFunction = self.modelConstFunBelow
#filterctfdata = scipy.ndimage.minimum_filter(ctfdata, size=2)
#for i in range(1):
# filterctfdata = (filterctfdata + scipy.ndimage.minimum_filter(filterctfdata, size=2))/2.0
#firstmin = filterctfdata[0:250].min()
#filterctfdata = numpy.where(filterctfdata>firstmin, firstmin, filterctfdata)
#filterctfdata = self.downwardRightMonotonicFilter(ctfdata)
filterctfdata = ctfdata
### run the initial minimizations
namelist, valuelist, fitparamslist = self.getAllInitialParameters(xdata,
filterctfdata, contraintFunction)
### figure out which initial fit was best
if self.debug is True:
namestr = "|"
valstr = "|"
conststr = "|"
for i in range(len(valuelist)):
constrainval = contraintFunction(fitparamslist[i], xdata, filterctfdata)
namestr += apDisplay.rightPadString("%s"%(namelist[i][:15]), 15)+"|"
valstr += apDisplay.leftPadString("%.4f"%(valuelist[i]), 15)+"|"
conststr += apDisplay.leftPadString("%.4e"%(constrainval), 15)+"|"
print namestr
print valstr
print conststr
### lowest is best
minvalindex = numpy.argmin(valuelist)
constrainval = contraintFunction(fitparamslist[minvalindex], xdata, filterctfdata)
valuelist = numpy.array(valuelist)
if contraint == "below":
minconval = -1e-2
elif contraint == "above":
minconval = -1e-4
else:
minconval = -1e-3
while constrainval < minconval and valuelist.min() < 1e6:
if constrainval < 0.1 and self.debug is True:
apDisplay.printMsg("Constraint violation: %.3e < %.3e"%(constrainval, minconval))
valuelist[minvalindex] *= 1e10
minvalindex = numpy.argmin(valuelist)
constrainval = contraintFunction(fitparamslist[minvalindex], xdata, filterctfdata)
if self.debug is True:
apDisplay.printColor( namelist[minvalindex]+" is best" , "cyan")
midfitparams = fitparamslist[minvalindex]
if self.debug is True:
print ( "middle parameters (%.5e, %.5e, %.5e, %.5e, %.5e)"
%(midfitparams[0], midfitparams[1], midfitparams[2], midfitparams[3], midfitparams[4]))
midvalue = self.modelFitFun(midfitparams, xdata, ctfdata)
if self.debug is True:
print "middle function value %.10f"%(midvalue)
constrainval = contraintFunction(midfitparams, xdata, ctfdata)
print "constrained value %.10e"%(constrainval)
### run the full minimization
rhobeg = (numpy.where(numpy.abs(midfitparams)<1e-20, 1e20, numpy.abs(midfitparams))).min()/1e7
if self.debug: print "RHO begin", rhobeg
fitparams = scipy.optimize.fmin_cobyla( self.modelFitFun, midfitparams,
args=(xdata, ctfdata), cons=[contraintFunction,],
consargs=(xdata, ctfdata), rhobeg=rhobeg, rhoend=rhobeg/1e4, iprint=0, maxfun=1e8)
if self.debug is True:
print ( "final parameters (%.4e, %.4e, %.4e, %.4e, %.4e)"
%(fitparams[0], fitparams[1], fitparams[2], fitparams[3], fitparams[4]))
finalvalue = self.modelFitFun(fitparams, xdata, ctfdata)
if self.debug is True:
print "final function value %.10f"%(finalvalue)
#writeDatFile("finalvalue.dat", fitparams, xdata, ctfdata)
if finalvalue <= midvalue:
if self.debug is True:
apDisplay.printColor("Final value is better", "green")
bestfitparams = fitparams
else:
if self.debug is True:
apDisplay.printColor("Final value is worse", "red")
bestfitparams = midfitparams
z = numpy.polyfit(xdata, filterctfdata, 3)
polyfitparams = [z[3], 0.0, z[2], z[1], z[0]]
if self.debug is True:
xdatasq = xdata**2
xdatasq = numpy.arange(0, len(xdata), 1)
from matplotlib import pyplot
pyplot.plot(xdatasq, ctfdata, 'r-', )
pyplot.plot(xdatasq, filterctfdata, 'b-', )
midfitdata = self.noiseModel(midfitparams, xdata)
pyplot.plot(xdatasq, midfitdata, 'm:', )
polyfitdata = self.noiseModel(polyfitparams, xdata)
pyplot.plot(xdatasq, polyfitdata, 'y-', )
finalfitdata = self.noiseModel(fitparams, xdata)
pyplot.plot(xdatasq, finalfitdata, 'k-', )
pyplot.show()
pyplot.clf()
"""
datadiff1 = scipy.ndimage.median_filter(numpy.diff(ctfdata), 3)
datadiff2 = scipy.ndimage.median_filter(numpy.diff(datadiff1), 27)
pyplot.plot(xdatasq[:500], (datadiff2/datadiff2.std())[:500], 'y-', )
pyplot.plot(xdatasq[:500], (ctfdata - ctfdata.mean())[:500], 'r-', )
pyplot.plot(xdatasq[:500], (datadiff1/datadiff1.std())[:500], 'c-', )
pyplot.show()
pyplot.clf()
"""
if self.debug is True:
apDisplay.printColor("Noise Model Complete in %s"
%(apDisplay.timeString(time.time()-t0)), "cyan")
return bestfitparams
def printCtfSummary(params, imgtree):
"""
prints a histogram of the best ctfvalues for the session
"""
# if there are no images in the imgtree, there was no new processing done, so exit this function early.
if not imgtree:
apDisplay.printWarning("There are no new results to summarize.")
return
sys.stderr.write("processing CTF histogram...\n")
### get best ctf values for each image
ctfhistconf = []
ctfhistval = []
for imgdata in imgtree:
if params[
'norejects'] is True and apDatabase.getSiblingImgAssessmentStatus(
imgdata) is False:
continue
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfvalues = ctfq.query()
### check if it has values
if ctfvalues is None:
continue
### find the best values
bestconf = 0.0
bestctfvalue = None
for ctfvalue in ctfvalues:
conf = calculateConfidenceScore(ctfvalue, False)
if conf > bestconf:
bestconf = conf
bestctfvalue = ctfvalue
ctfhistconf.append(bestconf)
ctfhistval.append(bestctfvalue)
ctfhistconf.sort()
confhist = {}
yspan = 20.0
minconf = ctfhistconf[0]
maxconf = ctfhistconf[len(ctfhistconf) - 1]
maxcount = 0
for conf in ctfhistconf:
c2 = round(conf * yspan, 0) / float(yspan)
if c2 in confhist:
confhist[c2] += 1
if confhist[c2] > maxcount:
maxcount = confhist[c2]
else:
confhist[c2] = 1
if maxcount > 70:
scale = 70.0 / float(maxcount)
sys.stderr.write(" * = " + str(round(scale, 1)) + " images\n")
else:
scale = 1.0
colorstr = {}
for i in range(int(yspan + 1)):
j = float(i) / yspan
if j < 0.5:
colorstr[j] = "red"
elif j < 0.8:
colorstr[j] = "yellow"
else:
colorstr[j] = "green"
sys.stderr.write("Confidence histogram:\n")
for i in range(int(yspan + 1)):
j = float(i) / yspan
if j < minconf - 1.0 / yspan:
continue
jstr = "%1.2f" % j
jstr = apDisplay.rightPadString(jstr, 5)
sys.stderr.write(jstr + "> ")
if j in confhist:
for k in range(int(confhist[j] * scale)):
sys.stderr.write(apDisplay.color("*", colorstr[j]))
sys.stderr.write("\n")
def printCtfSummary(params, imgtree):
"""
prints a histogram of the best ctfvalues for the session
"""
# if there are no images in the imgtree, there was no new processing done, so exit this function early.
if not imgtree:
apDisplay.printWarning("There are no new results to summarize.")
return
sys.stderr.write("processing CTF histogram...\n")
### get best ctf values for each image
ctfhistconf = []
ctfhistval = []
for imgdata in imgtree:
if params['norejects'] is True and apDatabase.getSiblingImgAssessmentStatus(imgdata) is False:
continue
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfvalues = ctfq.query()
### check if it has values
if ctfvalues is None:
continue
### find the best values
bestconf = 0.0
bestctfvalue = None
for ctfvalue in ctfvalues:
conf = calculateConfidenceScore(ctfvalue,False)
if conf > bestconf:
bestconf = conf
bestctfvalue = ctfvalue
ctfhistconf.append(bestconf)
ctfhistval.append(bestctfvalue)
ctfhistconf.sort()
confhist = {}
yspan = 20.0
minconf = ctfhistconf[0]
maxconf = ctfhistconf[len(ctfhistconf)-1]
maxcount = 0
for conf in ctfhistconf:
c2 = round(conf*yspan,0)/float(yspan)
if c2 in confhist:
confhist[c2] += 1
if confhist[c2] > maxcount:
maxcount = confhist[c2]
else:
confhist[c2] = 1
if maxcount > 70:
scale = 70.0/float(maxcount)
sys.stderr.write(" * = "+str(round(scale,1))+" images\n")
else:
scale = 1.0
colorstr = {}
for i in range(int(yspan+1)):
j = float(i)/yspan
if j < 0.5:
colorstr[j] = "red"
elif j < 0.8:
colorstr[j] = "yellow"
else:
colorstr[j] = "green"
sys.stderr.write("Confidence histogram:\n")
for i in range(int(yspan+1)):
j = float(i)/yspan
if j < minconf-1.0/yspan:
continue
jstr = "%1.2f" % j
jstr = apDisplay.rightPadString(jstr,5)
sys.stderr.write(jstr+"> ")
if j in confhist:
for k in range(int(confhist[j]*scale)):
sys.stderr.write(apDisplay.color("*",colorstr[j]))
sys.stderr.write("\n")
def modelCTFNoise(self, xdata, ctfdata, contraint="below"):
"""
Master control function to fit the CTF noise function
xdata - should be in inverse Angstroms
"""
t0 = time.time()
### need to reduce precision of the xdata
### otherwise it takes too long, with no better of a fit
xdata = xdata.astype(numpy.float32)
if self.debug is True:
apDisplay.printColor("CTF limits %.1f A -->> %.1fA"
%(1./xdata.min(), 1./xdata.max()), "cyan")
if contraint == "above":
if self.debug is True:
print "constrained above function"
contraintFunction = self.modelConstFunAbove
#filterctfdata = scipy.ndimage.maximum_filter(ctfdata, size=2)
#for i in range(1):
# filterctfdata = (filterctfdata + scipy.ndimage.maximum_filter(filterctfdata, size=2))/2.0
#firstmax = filterctfdata[0:250].max()
#filterctfdata = numpy.where(filterctfdata>firstmax, firstmax, filterctfdata)
#filterctfdata = self.upwardLeftMonotonicFilter(ctfdata)
filterctfdata = ctfdata
else:
if self.debug is True:
print "constrained below function"
contraintFunction = self.modelConstFunBelow
#filterctfdata = scipy.ndimage.minimum_filter(ctfdata, size=2)
#for i in range(1):
# filterctfdata = (filterctfdata + scipy.ndimage.minimum_filter(filterctfdata, size=2))/2.0
#firstmin = filterctfdata[0:250].min()
#filterctfdata = numpy.where(filterctfdata>firstmin, firstmin, filterctfdata)
#filterctfdata = self.downwardRightMonotonicFilter(ctfdata)
filterctfdata = ctfdata
### run the initial minimizations
namelist, valuelist, fitparamslist = self.getAllInitialParameters(xdata,
filterctfdata, contraintFunction)
### figure out which initial fit was best
if self.debug is True:
namestr = "|"
valstr = "|"
conststr = "|"
for i in range(len(valuelist)):
constrainval = contraintFunction(fitparamslist[i], xdata, filterctfdata)
namestr += apDisplay.rightPadString("%s"%(namelist[i][:15]), 15)+"|"
valstr += apDisplay.leftPadString("%.4f"%(valuelist[i]), 15)+"|"
conststr += apDisplay.leftPadString("%.4e"%(constrainval), 15)+"|"
print namestr
print valstr
print conststr
### lowest is best
minvalindex = numpy.argmin(valuelist)
constrainval = contraintFunction(fitparamslist[minvalindex], xdata, filterctfdata)
valuelist = numpy.array(valuelist)
if contraint == "below":
minconval = -1e-2
elif contraint == "above":
minconval = -1e-4
else:
minconval = -1e-3
while constrainval < minconval and valuelist.min() < 1e6:
if constrainval < 0.1 and self.debug is True:
apDisplay.printMsg("Constraint violation: %.3e < %.3e"%(constrainval, minconval))
valuelist[minvalindex] *= 1e10
minvalindex = numpy.argmin(valuelist)
constrainval = contraintFunction(fitparamslist[minvalindex], xdata, filterctfdata)
if self.debug is True:
apDisplay.printColor( namelist[minvalindex]+" is best" , "cyan")
midfitparams = fitparamslist[minvalindex]
if self.debug is True:
print ( "middle parameters (%.5e, %.5e, %.5e, %.5e, %.5e)"
%(midfitparams[0], midfitparams[1], midfitparams[2], midfitparams[3], midfitparams[4]))
midvalue = self.modelFitFun(midfitparams, xdata, ctfdata)
if self.debug is True:
print "middle function value %.10f"%(midvalue)
constrainval = contraintFunction(midfitparams, xdata, ctfdata)
print "constrained value %.10e"%(constrainval)
### run the full minimization
rhobeg = (numpy.where(numpy.abs(midfitparams)<1e-20, 1e20, numpy.abs(midfitparams))).min()/1e7
if self.debug: print "RHO begin", rhobeg
fitparams = scipy.optimize.fmin_cobyla( self.modelFitFun, midfitparams,
args=(xdata, ctfdata), cons=[contraintFunction,],
consargs=(xdata, ctfdata), rhobeg=rhobeg, rhoend=rhobeg/1e4, iprint=0, maxfun=1e8)
if self.debug is True:
print ( "final parameters (%.4e, %.4e, %.4e, %.4e, %.4e)"
%(fitparams[0], fitparams[1], fitparams[2], fitparams[3], fitparams[4]))
finalvalue = self.modelFitFun(fitparams, xdata, ctfdata)
if self.debug is True:
print "final function value %.10f"%(finalvalue)
#writeDatFile("finalvalue.dat", fitparams, xdata, ctfdata)
if finalvalue <= midvalue:
if self.debug is True:
apDisplay.printColor("Final value is better", "green")
bestfitparams = fitparams
else:
if self.debug is True:
apDisplay.printColor("Final value is worse", "red")
bestfitparams = midfitparams
z = numpy.polyfit(xdata, filterctfdata, 3)
polyfitparams = [z[3], 0.0, z[2], z[1], z[0]]
if self.debug is True:
xdatasq = xdata**2
xdatasq = numpy.arange(0, len(xdata), 1)
from matplotlib import pyplot
pyplot.plot(xdatasq, ctfdata, 'r-', )
pyplot.plot(xdatasq, filterctfdata, 'b-', )
midfitdata = self.noiseModel(midfitparams, xdata)
pyplot.plot(xdatasq, midfitdata, 'm:', )
polyfitdata = self.noiseModel(polyfitparams, xdata)
pyplot.plot(xdatasq, polyfitdata, 'y-', )
finalfitdata = self.noiseModel(fitparams, xdata)
pyplot.plot(xdatasq, finalfitdata, 'k-', )
pyplot.show()
pyplot.clf()
"""
datadiff1 = scipy.ndimage.median_filter(numpy.diff(ctfdata), 3)
datadiff2 = scipy.ndimage.median_filter(numpy.diff(datadiff1), 27)
pyplot.plot(xdatasq[:500], (datadiff2/datadiff2.std())[:500], 'y-', )
pyplot.plot(xdatasq[:500], (ctfdata - ctfdata.mean())[:500], 'r-', )
pyplot.plot(xdatasq[:500], (datadiff1/datadiff1.std())[:500], 'c-', )
pyplot.show()
pyplot.clf()
"""
if self.debug is True:
apDisplay.printColor("Noise Model Complete in %s"
%(apDisplay.timeString(time.time()-t0)), "cyan")
return bestfitparams