def spiderOutLine(num, floatlist):
line = "%04d" % num
line += " %1d" % len(floatlist)
for fnum in floatlist:
line += " "+apDisplay.leftPadString("%3.6f" % fnum, n=11)
line += "\n"
return line
def colorTupleToHex(r, g, b, scale=1):
#print r,g,b
ir = int(r*scale)
ig = int(g*scale)
ib = int(b*scale)
hexcode = str(hex(ir*256**2 + ig*256 + ib))[2:]
hexstr = "#"+apDisplay.leftPadString(hexcode, n=6, fill='0')
return hexstr
def spiderOutputLine(int1, int2, float1, float2, float3, float4, float5, float6=1.0):
line = "%04d" % int1
line += " %1d" % int2
line += " "+apDisplay.leftPadString("%3.6f" % float1, n=11)
line += " "+apDisplay.leftPadString("%3.6f" % float2, n=11)
line += " "+apDisplay.leftPadString("%3.6f" % float3, n=11)
line += " "+apDisplay.leftPadString("%3.6f" % float4, n=11)
line += " "+apDisplay.leftPadString("%3.6f" % float5, n=11)
line += " "+apDisplay.leftPadString("%3.6f" % float6, n=11)
line += "\n"
return line
def spiderOutputLine(int1, int2, float1, float2, float3, float4, float5, float6=1.0):
line = "%04d" % int1
line += " %1d" % int2
line += " "+apDisplay.leftPadString("%3.6f" % float1, n=11)
line += " "+apDisplay.leftPadString("%3.6f" % float2, n=11)
line += " "+apDisplay.leftPadString("%3.6f" % float3, n=11)
line += " "+apDisplay.leftPadString("%3.6f" % float4, n=11)
line += " "+apDisplay.leftPadString("%3.6f" % float5, n=11)
line += " "+apDisplay.leftPadString("%3.6f" % float6, n=11)
line += "\n"
return line
def createSplitParamFiles(self, paramfile):
apDisplay.printColor("Splitting parameter file for gold standard", "purple")
origF = open(paramfile, "r")
leftF = open("params.left.iter00.par", "w")
rightF = open("params.right.iter00.par", "w")
for line in origF:
partNum = int(line[:7])
data = line[7:].rstrip()
if partNum % 2 == 0:
newPartNum = partNum/2
else:
newPartNum = (partNum+1)/2
newLine = "%s%s"%(apDisplay.leftPadString("%d"%(newPartNum), 7), data)
if partNum % 2 == 0:
rightF.write(newLine+'\n')
else:
leftF.write(newLine+'\n')
origF.close()
rightF.close()
leftF.close()
return
def createSplitParamFiles(self, paramfile):
apDisplay.printColor("Splitting parameter file for gold standard",
"purple")
origF = open(paramfile, "r")
leftF = open("params.left.iter00.par", "w")
rightF = open("params.right.iter00.par", "w")
for line in origF:
partNum = int(line[:7])
data = line[7:].rstrip()
if partNum % 2 == 0:
newPartNum = partNum / 2
else:
newPartNum = (partNum + 1) / 2
newLine = "%s%s" % (apDisplay.leftPadString(
"%d" % (newPartNum), 7), data)
if partNum % 2 == 0:
rightF.write(newLine + '\n')
else:
leftF.write(newLine + '\n')
origF.close()
rightF.close()
leftF.close()
return
def modelCTFNoise(self, xdata, ctfdata, contraint="below"):
"""
Master control function to fit the CTF noise function
xdata - should be in inverse Angstroms
"""
t0 = time.time()
### need to reduce precision of the xdata
### otherwise it takes too long, with no better of a fit
xdata = xdata.astype(numpy.float32)
if self.debug is True:
apDisplay.printColor("CTF limits %.1f A -->> %.1fA"
%(1./xdata.min(), 1./xdata.max()), "cyan")
if contraint == "above":
if self.debug is True:
print "constrained above function"
contraintFunction = self.modelConstFunAbove
#filterctfdata = scipy.ndimage.maximum_filter(ctfdata, size=2)
#for i in range(1):
# filterctfdata = (filterctfdata + scipy.ndimage.maximum_filter(filterctfdata, size=2))/2.0
#firstmax = filterctfdata[0:250].max()
#filterctfdata = numpy.where(filterctfdata>firstmax, firstmax, filterctfdata)
#filterctfdata = self.upwardLeftMonotonicFilter(ctfdata)
filterctfdata = ctfdata
else:
if self.debug is True:
print "constrained below function"
contraintFunction = self.modelConstFunBelow
#filterctfdata = scipy.ndimage.minimum_filter(ctfdata, size=2)
#for i in range(1):
# filterctfdata = (filterctfdata + scipy.ndimage.minimum_filter(filterctfdata, size=2))/2.0
#firstmin = filterctfdata[0:250].min()
#filterctfdata = numpy.where(filterctfdata>firstmin, firstmin, filterctfdata)
#filterctfdata = self.downwardRightMonotonicFilter(ctfdata)
filterctfdata = ctfdata
### run the initial minimizations
namelist, valuelist, fitparamslist = self.getAllInitialParameters(xdata,
filterctfdata, contraintFunction)
### figure out which initial fit was best
if self.debug is True:
namestr = "|"
valstr = "|"
conststr = "|"
for i in range(len(valuelist)):
constrainval = contraintFunction(fitparamslist[i], xdata, filterctfdata)
namestr += apDisplay.rightPadString("%s"%(namelist[i][:15]), 15)+"|"
valstr += apDisplay.leftPadString("%.4f"%(valuelist[i]), 15)+"|"
conststr += apDisplay.leftPadString("%.4e"%(constrainval), 15)+"|"
print namestr
print valstr
print conststr
### lowest is best
minvalindex = numpy.argmin(valuelist)
constrainval = contraintFunction(fitparamslist[minvalindex], xdata, filterctfdata)
valuelist = numpy.array(valuelist)
if contraint == "below":
minconval = -1e-2
elif contraint == "above":
minconval = -1e-4
else:
minconval = -1e-3
while constrainval < minconval and valuelist.min() < 1e6:
if constrainval < 0.1 and self.debug is True:
apDisplay.printMsg("Constraint violation: %.3e < %.3e"%(constrainval, minconval))
valuelist[minvalindex] *= 1e10
minvalindex = numpy.argmin(valuelist)
constrainval = contraintFunction(fitparamslist[minvalindex], xdata, filterctfdata)
if self.debug is True:
apDisplay.printColor( namelist[minvalindex]+" is best" , "cyan")
midfitparams = fitparamslist[minvalindex]
if self.debug is True:
print ( "middle parameters (%.5e, %.5e, %.5e, %.5e, %.5e)"
%(midfitparams[0], midfitparams[1], midfitparams[2], midfitparams[3], midfitparams[4]))
midvalue = self.modelFitFun(midfitparams, xdata, ctfdata)
if self.debug is True:
print "middle function value %.10f"%(midvalue)
constrainval = contraintFunction(midfitparams, xdata, ctfdata)
print "constrained value %.10e"%(constrainval)
### run the full minimization
rhobeg = (numpy.where(numpy.abs(midfitparams)<1e-20, 1e20, numpy.abs(midfitparams))).min()/1e7
if self.debug: print "RHO begin", rhobeg
fitparams = scipy.optimize.fmin_cobyla( self.modelFitFun, midfitparams,
args=(xdata, ctfdata), cons=[contraintFunction,],
consargs=(xdata, ctfdata), rhobeg=rhobeg, rhoend=rhobeg/1e4, iprint=0, maxfun=1e8)
if self.debug is True:
print ( "final parameters (%.4e, %.4e, %.4e, %.4e, %.4e)"
%(fitparams[0], fitparams[1], fitparams[2], fitparams[3], fitparams[4]))
finalvalue = self.modelFitFun(fitparams, xdata, ctfdata)
if self.debug is True:
print "final function value %.10f"%(finalvalue)
#writeDatFile("finalvalue.dat", fitparams, xdata, ctfdata)
if finalvalue <= midvalue:
if self.debug is True:
apDisplay.printColor("Final value is better", "green")
bestfitparams = fitparams
else:
if self.debug is True:
apDisplay.printColor("Final value is worse", "red")
bestfitparams = midfitparams
z = numpy.polyfit(xdata, filterctfdata, 3)
polyfitparams = [z[3], 0.0, z[2], z[1], z[0]]
if self.debug is True:
xdatasq = xdata**2
xdatasq = numpy.arange(0, len(xdata), 1)
from matplotlib import pyplot
pyplot.plot(xdatasq, ctfdata, 'r-', )
pyplot.plot(xdatasq, filterctfdata, 'b-', )
midfitdata = self.noiseModel(midfitparams, xdata)
pyplot.plot(xdatasq, midfitdata, 'm:', )
polyfitdata = self.noiseModel(polyfitparams, xdata)
pyplot.plot(xdatasq, polyfitdata, 'y-', )
finalfitdata = self.noiseModel(fitparams, xdata)
pyplot.plot(xdatasq, finalfitdata, 'k-', )
pyplot.show()
pyplot.clf()
"""
datadiff1 = scipy.ndimage.median_filter(numpy.diff(ctfdata), 3)
datadiff2 = scipy.ndimage.median_filter(numpy.diff(datadiff1), 27)
pyplot.plot(xdatasq[:500], (datadiff2/datadiff2.std())[:500], 'y-', )
pyplot.plot(xdatasq[:500], (ctfdata - ctfdata.mean())[:500], 'r-', )
pyplot.plot(xdatasq[:500], (datadiff1/datadiff1.std())[:500], 'c-', )
pyplot.show()
pyplot.clf()
"""
if self.debug is True:
apDisplay.printColor("Noise Model Complete in %s"
%(apDisplay.timeString(time.time()-t0)), "cyan")
return bestfitparams
def modelCTFNoise(self, xdata, ctfdata, contraint="below"):
"""
Master control function to fit the CTF noise function
xdata - should be in inverse Angstroms
"""
t0 = time.time()
### need to reduce precision of the xdata
### otherwise it takes too long, with no better of a fit
xdata = xdata.astype(numpy.float32)
if self.debug is True:
apDisplay.printColor("CTF limits %.1f A -->> %.1fA"
%(1./xdata.min(), 1./xdata.max()), "cyan")
if contraint == "above":
if self.debug is True:
print "constrained above function"
contraintFunction = self.modelConstFunAbove
#filterctfdata = scipy.ndimage.maximum_filter(ctfdata, size=2)
#for i in range(1):
# filterctfdata = (filterctfdata + scipy.ndimage.maximum_filter(filterctfdata, size=2))/2.0
#firstmax = filterctfdata[0:250].max()
#filterctfdata = numpy.where(filterctfdata>firstmax, firstmax, filterctfdata)
#filterctfdata = self.upwardLeftMonotonicFilter(ctfdata)
filterctfdata = ctfdata
else:
if self.debug is True:
print "constrained below function"
contraintFunction = self.modelConstFunBelow
#filterctfdata = scipy.ndimage.minimum_filter(ctfdata, size=2)
#for i in range(1):
# filterctfdata = (filterctfdata + scipy.ndimage.minimum_filter(filterctfdata, size=2))/2.0
#firstmin = filterctfdata[0:250].min()
#filterctfdata = numpy.where(filterctfdata>firstmin, firstmin, filterctfdata)
#filterctfdata = self.downwardRightMonotonicFilter(ctfdata)
filterctfdata = ctfdata
### run the initial minimizations
namelist, valuelist, fitparamslist = self.getAllInitialParameters(xdata,
filterctfdata, contraintFunction)
### figure out which initial fit was best
if self.debug is True:
namestr = "|"
valstr = "|"
conststr = "|"
for i in range(len(valuelist)):
constrainval = contraintFunction(fitparamslist[i], xdata, filterctfdata)
namestr += apDisplay.rightPadString("%s"%(namelist[i][:15]), 15)+"|"
valstr += apDisplay.leftPadString("%.4f"%(valuelist[i]), 15)+"|"
conststr += apDisplay.leftPadString("%.4e"%(constrainval), 15)+"|"
print namestr
print valstr
print conststr
### lowest is best
minvalindex = numpy.argmin(valuelist)
constrainval = contraintFunction(fitparamslist[minvalindex], xdata, filterctfdata)
valuelist = numpy.array(valuelist)
if contraint == "below":
minconval = -1e-2
elif contraint == "above":
minconval = -1e-4
else:
minconval = -1e-3
while constrainval < minconval and valuelist.min() < 1e6:
if constrainval < 0.1 and self.debug is True:
apDisplay.printMsg("Constraint violation: %.3e < %.3e"%(constrainval, minconval))
valuelist[minvalindex] *= 1e10
minvalindex = numpy.argmin(valuelist)
constrainval = contraintFunction(fitparamslist[minvalindex], xdata, filterctfdata)
if self.debug is True:
apDisplay.printColor( namelist[minvalindex]+" is best" , "cyan")
midfitparams = fitparamslist[minvalindex]
if self.debug is True:
print ( "middle parameters (%.5e, %.5e, %.5e, %.5e, %.5e)"
%(midfitparams[0], midfitparams[1], midfitparams[2], midfitparams[3], midfitparams[4]))
midvalue = self.modelFitFun(midfitparams, xdata, ctfdata)
if self.debug is True:
print "middle function value %.10f"%(midvalue)
constrainval = contraintFunction(midfitparams, xdata, ctfdata)
print "constrained value %.10e"%(constrainval)
### run the full minimization
rhobeg = (numpy.where(numpy.abs(midfitparams)<1e-20, 1e20, numpy.abs(midfitparams))).min()/1e7
if self.debug: print "RHO begin", rhobeg
fitparams = scipy.optimize.fmin_cobyla( self.modelFitFun, midfitparams,
args=(xdata, ctfdata), cons=[contraintFunction,],
consargs=(xdata, ctfdata), rhobeg=rhobeg, rhoend=rhobeg/1e4, iprint=0, maxfun=1e8)
if self.debug is True:
print ( "final parameters (%.4e, %.4e, %.4e, %.4e, %.4e)"
%(fitparams[0], fitparams[1], fitparams[2], fitparams[3], fitparams[4]))
finalvalue = self.modelFitFun(fitparams, xdata, ctfdata)
if self.debug is True:
print "final function value %.10f"%(finalvalue)
#writeDatFile("finalvalue.dat", fitparams, xdata, ctfdata)
if finalvalue <= midvalue:
if self.debug is True:
apDisplay.printColor("Final value is better", "green")
bestfitparams = fitparams
else:
if self.debug is True:
apDisplay.printColor("Final value is worse", "red")
bestfitparams = midfitparams
z = numpy.polyfit(xdata, filterctfdata, 3)
polyfitparams = [z[3], 0.0, z[2], z[1], z[0]]
if self.debug is True:
xdatasq = xdata**2
xdatasq = numpy.arange(0, len(xdata), 1)
from matplotlib import pyplot
pyplot.plot(xdatasq, ctfdata, 'r-', )
pyplot.plot(xdatasq, filterctfdata, 'b-', )
midfitdata = self.noiseModel(midfitparams, xdata)
pyplot.plot(xdatasq, midfitdata, 'm:', )
polyfitdata = self.noiseModel(polyfitparams, xdata)
pyplot.plot(xdatasq, polyfitdata, 'y-', )
finalfitdata = self.noiseModel(fitparams, xdata)
pyplot.plot(xdatasq, finalfitdata, 'k-', )
pyplot.show()
pyplot.clf()
"""
datadiff1 = scipy.ndimage.median_filter(numpy.diff(ctfdata), 3)
datadiff2 = scipy.ndimage.median_filter(numpy.diff(datadiff1), 27)
pyplot.plot(xdatasq[:500], (datadiff2/datadiff2.std())[:500], 'y-', )
pyplot.plot(xdatasq[:500], (ctfdata - ctfdata.mean())[:500], 'r-', )
pyplot.plot(xdatasq[:500], (datadiff1/datadiff1.std())[:500], 'c-', )
pyplot.show()
pyplot.clf()
"""
if self.debug is True:
apDisplay.printColor("Noise Model Complete in %s"
%(apDisplay.timeString(time.time()-t0)), "cyan")
return bestfitparams