def fullTriSectionNormalize(self, raddata, PSD, defocus):
t0 = time.time()
###
### PART 1: BACKGROUND NOISE SUBTRACTION
###
# skip the center
valleys = ctftools.getCtfExtrema(defocus, self.freq*1e10,
self.ctfvalues['cs'], self.ctfvalues['volts'], self.ctfvalues['amplitude_contrast'],
numzeros=250, zerotype="valleys")
firstvalley = valleys[0]
valleyradii = numpy.array(valleys, dtype=numpy.float64)*self.freq
firstvalleyindex = numpy.searchsorted(raddata, self.freq*firstvalley)
apDisplay.printColor("First valley: %.1f -> %d (1/%.1f A)"
%(firstvalley, firstvalleyindex, 1/(firstvalley*self.freq)), "yellow")
### split the function up in first 3/5 and last 3/5 of data with 1/5 overlap
numpoints = len(raddata) - firstvalleyindex
part1start = firstvalleyindex
part1end = int(firstvalleyindex + numpoints*6/10.)
part2start = int(firstvalleyindex + numpoints*5/10.)
part2end = int(firstvalleyindex + numpoints*9/10.)
part3start = int(firstvalleyindex + numpoints*8/10.)
part3end = len(raddata)
CtfNoise = ctfnoise.CtfNoise()
if valleyradii is None:
valleydata = ctfnoise.peakExtender(raddata, PSD, valleyradii, "below")
else:
valleydata = ndimage.minimum_filter(PSD, 4)
## first part data
noisefitparams1 = CtfNoise.modelCTFNoise(raddata[part1start:part1end],
valleydata[part1start:part1end], "below")
noisedata1 = CtfNoise.noiseModel(noisefitparams1, raddata)
## second part data
noisefitparams2 = CtfNoise.modelCTFNoise(raddata[part2start:part2end],
valleydata[part2start:part2end], "below")
noisedata2 = CtfNoise.noiseModel(noisefitparams2, raddata)
## third part data
noisefitparams3 = CtfNoise.modelCTFNoise(raddata[part3start:part3end],
valleydata[part3start:part3end], "below")
noisedata3 = CtfNoise.noiseModel(noisefitparams3, raddata)
## merge data
scale = numpy.arange(part1end-part2start, dtype=numpy.float32)
scale /= scale.max()
overlapdata1 = noisedata1[part2start:part1end]*(1-scale) + noisedata2[part2start:part1end]*scale
scale = numpy.arange(part2end-part3start, dtype=numpy.float32)
scale /= scale.max()
overlapdata2 = noisedata2[part3start:part2end]*(1-scale) + noisedata3[part3start:part2end]*scale
mergedata = numpy.hstack((noisedata1[:part2start], overlapdata1,
noisedata2[part1end:part3start], overlapdata2,
noisedata3[part2end:]))
noisedata = mergedata
### DO THE SUBTRACTION
normexpPSD = numpy.exp(PSD) - numpy.exp(noisedata)
normlogPSD = numpy.log(numpy.where(normexpPSD<1, 1, normexpPSD))
###
### PART 2: ENVELOPE NORMALIZATION
###
# high pass filter the center
peaks = ctftools.getCtfExtrema(defocus, self.freq*1e10,
self.ctfvalues['cs'], self.ctfvalues['volts'], self.ctfvalues['amplitude_contrast'],
numzeros=250, zerotype="peaks")
firstpeak = peaks[0]
peakradii = numpy.array(peaks, dtype=numpy.float64)*self.freq
firstpeakindex = numpy.searchsorted(raddata, firstpeak*self.freq)
apDisplay.printColor("First peak: %.1f (1/%.1f A)"%
(firstpeakindex, 1/(firstpeak*self.freq)), "yellow")
### split the function up in first 3/5 and last 3/5 of data with 1/5 overlap
numpoints = len(raddata) - firstpeakindex
part1start = firstpeakindex
part1end = int(firstpeakindex + numpoints*6/10.)
part2start = int(firstpeakindex + numpoints*5/10.)
part2end = int(firstpeakindex + numpoints*9/10.)
part3start = int(firstpeakindex + numpoints*8/10.)
part3end = len(raddata)
CtfNoise = ctfnoise.CtfNoise()
if peakradii is None:
peakdata = ctfnoise.peakExtender(raddata, normlogPSD, peakradii, "above")
else:
peakdata = ndimage.maximum_filter(normlogPSD, 4)
## first part data
envelopfitparams1 = CtfNoise.modelCTFNoise(raddata[part1start:part1end],
peakdata[part1start:part1end], "above")
envelopdata1 = CtfNoise.noiseModel(envelopfitparams1, raddata)
## second part data
envelopfitparams2 = CtfNoise.modelCTFNoise(raddata[part2start:part2end],
peakdata[part2start:part2end], "above")
envelopdata2 = CtfNoise.noiseModel(envelopfitparams2, raddata)
## third part data
envelopfitparams3 = CtfNoise.modelCTFNoise(raddata[part3start:part3end],
peakdata[part3start:part3end], "above")
envelopdata3 = CtfNoise.noiseModel(envelopfitparams3, raddata)
## merge data
scale = numpy.arange(part1end-part2start, dtype=numpy.float32)
scale /= scale.max()
overlapdata1 = envelopdata1[part2start:part1end]*(1-scale) + envelopdata2[part2start:part1end]*scale
scale = numpy.arange(part2end-part3start, dtype=numpy.float32)
scale /= scale.max()
overlapdata2 = envelopdata2[part3start:part2end]*(1-scale) + envelopdata3[part3start:part2end]*scale
mergedata = numpy.hstack((envelopdata1[:part2start], overlapdata1,
envelopdata2[part1end:part3start], overlapdata2,
envelopdata3[part2end:]))
envelopdata = mergedata
normnormexpPSD = normexpPSD / numpy.exp(envelopdata)
if self.debug is True:
from matplotlib import pyplot
pyplot.clf()
pyplot.subplot(3,1,1)
raddatasq = raddata**2
pyplot.plot(raddatasq, normnormexpPSD, 'k.',)
a = pyplot.plot(raddatasq, PSD, 'k-', alpha=0.5)
a = pyplot.plot(raddatasq, valleydata, 'k-', alpha=0.5)
b = pyplot.plot(raddatasq[firstvalleyindex:], noisedata[firstvalleyindex:], '--', color="purple", linewidth=2)
c = pyplot.plot(raddatasq[part1start:part1end],
noisedata1[part1start:part1end], 'b-', alpha=0.5, linewidth=2)
d = pyplot.plot(raddatasq[part2start:part2end],
noisedata2[part2start:part2end], 'r-', alpha=0.5, linewidth=2)
e = pyplot.plot(raddatasq[part3start:part3end],
noisedata3[part3start:part3end], '-', alpha=0.5, linewidth=2, color="green")
pyplot.legend([a, b, c, d, e], ["data", "merge", "part 1", "part 2", "part 3"])
pyplot.xlim(xmax=raddatasq.max())
pyplot.ylim(ymin=noisedata.min(), ymax=PSD[part1start:].max())
pyplot.subplot(3,1,2)
a = pyplot.plot(raddatasq, normlogPSD, 'k.',)
a = pyplot.plot(raddatasq, normlogPSD, 'k-', alpha=0.5)
a = pyplot.plot(raddatasq, peakdata, 'k-', alpha=0.5)
b = pyplot.plot(raddatasq, mergedata, '--', color="purple", linewidth=2)
c = pyplot.plot(raddatasq[part1start:part1end],
envelopdata1[part1start:part1end], 'b-', alpha=0.5, linewidth=2)
d = pyplot.plot(raddatasq[part2start:part2end],
envelopdata2[part2start:part2end], 'r-', alpha=0.5, linewidth=2)
e = pyplot.plot(raddatasq[part3start:part3end],
envelopdata3[part3start:part3end], '-', alpha=0.5, linewidth=2, color="green")
pyplot.legend([a, b, c, d, e], ["data", "merge", "part 1", "part 2", "part 3"])
pyplot.xlim(xmax=raddatasq.max())
pyplot.ylim(ymin=normlogPSD[part1start:].min(), ymax=envelopdata.max())
pyplot.subplot(3,1,3)
pyplot.plot(raddatasq, normnormexpPSD, 'k.',)
pyplot.plot(raddatasq, normnormexpPSD, 'k-', alpha=0.5)
pyplot.xlim(xmax=raddatasq.max())
pyplot.subplots_adjust(wspace=0.05, hspace=0.05,
bottom=0.05, left=0.05, top=0.95, right=0.95, )
pyplot.show()
apDisplay.printColor("TriSection complete in %s"
%(apDisplay.timeString(time.time()-t0)), "cyan")
return normnormexpPSD
def fullTriSectionNormalize(self, raddata, PSD, defocus):
t0 = time.time()
###
### PART 1: BACKGROUND NOISE SUBTRACTION
###
# skip the center
valleys = ctftools.getCtfExtrema(defocus,
self.freq * 1e10,
self.ctfvalues['cs'],
self.ctfvalues['volts'],
self.ctfvalues['amplitude_contrast'],
numzeros=250,
zerotype="valleys")
firstvalley = valleys[0]
valleyradii = numpy.array(valleys, dtype=numpy.float64) * self.freq
firstvalleyindex = numpy.searchsorted(raddata, self.freq * firstvalley)
apDisplay.printColor(
"First valley: %.1f -> %d (1/%.1f A)" %
(firstvalley, firstvalleyindex, 1 / (firstvalley * self.freq)),
"yellow")
### split the function up in first 3/5 and last 3/5 of data with 1/5 overlap
numpoints = len(raddata) - firstvalleyindex
part1start = firstvalleyindex
part1end = int(firstvalleyindex + numpoints * 6 / 10.)
part2start = int(firstvalleyindex + numpoints * 5 / 10.)
part2end = int(firstvalleyindex + numpoints * 9 / 10.)
part3start = int(firstvalleyindex + numpoints * 8 / 10.)
part3end = len(raddata)
CtfNoise = ctfnoise.CtfNoise()
if valleyradii is None:
valleydata = ctfnoise.peakExtender(raddata, PSD, valleyradii,
"below")
else:
valleydata = ndimage.minimum_filter(PSD, 4)
## first part data
noisefitparams1 = CtfNoise.modelCTFNoise(
raddata[part1start:part1end], valleydata[part1start:part1end],
"below")
noisedata1 = CtfNoise.noiseModel(noisefitparams1, raddata)
## second part data
noisefitparams2 = CtfNoise.modelCTFNoise(
raddata[part2start:part2end], valleydata[part2start:part2end],
"below")
noisedata2 = CtfNoise.noiseModel(noisefitparams2, raddata)
## third part data
noisefitparams3 = CtfNoise.modelCTFNoise(
raddata[part3start:part3end], valleydata[part3start:part3end],
"below")
noisedata3 = CtfNoise.noiseModel(noisefitparams3, raddata)
## merge data
scale = numpy.arange(part1end - part2start, dtype=numpy.float32)
scale /= scale.max()
overlapdata1 = noisedata1[part2start:part1end] * (
1 - scale) + noisedata2[part2start:part1end] * scale
scale = numpy.arange(part2end - part3start, dtype=numpy.float32)
scale /= scale.max()
overlapdata2 = noisedata2[part3start:part2end] * (
1 - scale) + noisedata3[part3start:part2end] * scale
mergedata = numpy.hstack((noisedata1[:part2start], overlapdata1,
noisedata2[part1end:part3start],
overlapdata2, noisedata3[part2end:]))
noisedata = mergedata
### DO THE SUBTRACTION
normexpPSD = numpy.exp(PSD) - numpy.exp(noisedata)
normlogPSD = numpy.log(numpy.where(normexpPSD < 1, 1, normexpPSD))
###
### PART 2: ENVELOPE NORMALIZATION
###
# high pass filter the center
peaks = ctftools.getCtfExtrema(defocus,
self.freq * 1e10,
self.ctfvalues['cs'],
self.ctfvalues['volts'],
self.ctfvalues['amplitude_contrast'],
numzeros=250,
zerotype="peaks")
firstpeak = peaks[0]
peakradii = numpy.array(peaks, dtype=numpy.float64) * self.freq
firstpeakindex = numpy.searchsorted(raddata, firstpeak * self.freq)
apDisplay.printColor(
"First peak: %.1f (1/%.1f A)" %
(firstpeakindex, 1 / (firstpeak * self.freq)), "yellow")
### split the function up in first 3/5 and last 3/5 of data with 1/5 overlap
numpoints = len(raddata) - firstpeakindex
part1start = firstpeakindex
part1end = int(firstpeakindex + numpoints * 6 / 10.)
part2start = int(firstpeakindex + numpoints * 5 / 10.)
part2end = int(firstpeakindex + numpoints * 9 / 10.)
part3start = int(firstpeakindex + numpoints * 8 / 10.)
part3end = len(raddata)
CtfNoise = ctfnoise.CtfNoise()
if peakradii is None:
peakdata = ctfnoise.peakExtender(raddata, normlogPSD, peakradii,
"above")
else:
peakdata = ndimage.maximum_filter(normlogPSD, 4)
## first part data
envelopfitparams1 = CtfNoise.modelCTFNoise(
raddata[part1start:part1end], peakdata[part1start:part1end],
"above")
envelopdata1 = CtfNoise.noiseModel(envelopfitparams1, raddata)
## second part data
envelopfitparams2 = CtfNoise.modelCTFNoise(
raddata[part2start:part2end], peakdata[part2start:part2end],
"above")
envelopdata2 = CtfNoise.noiseModel(envelopfitparams2, raddata)
## third part data
envelopfitparams3 = CtfNoise.modelCTFNoise(
raddata[part3start:part3end], peakdata[part3start:part3end],
"above")
envelopdata3 = CtfNoise.noiseModel(envelopfitparams3, raddata)
## merge data
scale = numpy.arange(part1end - part2start, dtype=numpy.float32)
scale /= scale.max()
overlapdata1 = envelopdata1[part2start:part1end] * (
1 - scale) + envelopdata2[part2start:part1end] * scale
scale = numpy.arange(part2end - part3start, dtype=numpy.float32)
scale /= scale.max()
overlapdata2 = envelopdata2[part3start:part2end] * (
1 - scale) + envelopdata3[part3start:part2end] * scale
mergedata = numpy.hstack((envelopdata1[:part2start], overlapdata1,
envelopdata2[part1end:part3start],
overlapdata2, envelopdata3[part2end:]))
envelopdata = mergedata
normnormexpPSD = normexpPSD / numpy.exp(envelopdata)
if self.debug is True:
from matplotlib import pyplot
pyplot.clf()
pyplot.subplot(3, 1, 1)
raddatasq = raddata**2
pyplot.plot(
raddatasq,
normnormexpPSD,
'k.',
)
a = pyplot.plot(raddatasq, PSD, 'k-', alpha=0.5)
a = pyplot.plot(raddatasq, valleydata, 'k-', alpha=0.5)
b = pyplot.plot(raddatasq[firstvalleyindex:],
noisedata[firstvalleyindex:],
'--',
color="purple",
linewidth=2)
c = pyplot.plot(raddatasq[part1start:part1end],
noisedata1[part1start:part1end],
'b-',
alpha=0.5,
linewidth=2)
d = pyplot.plot(raddatasq[part2start:part2end],
noisedata2[part2start:part2end],
'r-',
alpha=0.5,
linewidth=2)
e = pyplot.plot(raddatasq[part3start:part3end],
noisedata3[part3start:part3end],
'-',
alpha=0.5,
linewidth=2,
color="green")
pyplot.legend([a, b, c, d, e],
["data", "merge", "part 1", "part 2", "part 3"])
pyplot.xlim(xmax=raddatasq.max())
pyplot.ylim(ymin=noisedata.min(), ymax=PSD[part1start:].max())
pyplot.subplot(3, 1, 2)
a = pyplot.plot(
raddatasq,
normlogPSD,
'k.',
)
a = pyplot.plot(raddatasq, normlogPSD, 'k-', alpha=0.5)
a = pyplot.plot(raddatasq, peakdata, 'k-', alpha=0.5)
b = pyplot.plot(raddatasq,
mergedata,
'--',
color="purple",
linewidth=2)
c = pyplot.plot(raddatasq[part1start:part1end],
envelopdata1[part1start:part1end],
'b-',
alpha=0.5,
linewidth=2)
d = pyplot.plot(raddatasq[part2start:part2end],
envelopdata2[part2start:part2end],
'r-',
alpha=0.5,
linewidth=2)
e = pyplot.plot(raddatasq[part3start:part3end],
envelopdata3[part3start:part3end],
'-',
alpha=0.5,
linewidth=2,
color="green")
pyplot.legend([a, b, c, d, e],
["data", "merge", "part 1", "part 2", "part 3"])
pyplot.xlim(xmax=raddatasq.max())
pyplot.ylim(ymin=normlogPSD[part1start:].min(),
ymax=envelopdata.max())
pyplot.subplot(3, 1, 3)
pyplot.plot(
raddatasq,
normnormexpPSD,
'k.',
)
pyplot.plot(raddatasq, normnormexpPSD, 'k-', alpha=0.5)
pyplot.xlim(xmax=raddatasq.max())
pyplot.subplots_adjust(
wspace=0.05,
hspace=0.05,
bottom=0.05,
left=0.05,
top=0.95,
right=0.95,
)
pyplot.show()
apDisplay.printColor(
"TriSection complete in %s" %
(apDisplay.timeString(time.time() - t0)), "cyan")
return normnormexpPSD
def normalizeCtf(self, zdata2d, twod=True):
"""
inner cut radius - radius for number of pixels to clip in the center of image
"""
###
### PART 1: SETUP PARAMETERS AND ELLIPTICAL AVERAGE
###
apDisplay.printColor("PART 1: SETUP PARAMETERS AND ELLIPTICAL AVERAGE", "magenta")
meandefocus = math.sqrt(self.defocus1*self.defocus2)
if meandefocus < 0.6e-6:
self.ringwidth = 3.0
elif meandefocus < 1.0e-6:
self.ringwidth = 2.0
elif meandefocus > 5.0e-6:
self.ringwidth = 0.5
### get all peak (not valley)
peak = ctftools.getCtfExtrema(meandefocus, self.trimfreq*1e10, self.cs, self.volts,
self.ampcontrast, numzeros=250, zerotype="peak")
apDisplay.printMsg("Number of available peaks is %d"%(len(peak)))
if len(peak) < 6:
apDisplay.printWarning("Too few peaks to work with, probably bad defocus estimate")
return None
firstpeak = peak[0]
peakradii = numpy.array(peak, dtype=numpy.float64)*self.trimfreq
### get all valley (not peak)
valley = ctftools.getCtfExtrema(meandefocus, self.trimfreq*1e10, self.cs, self.volts,
self.ampcontrast, numzeros=250, zerotype="valley")
firstvalley = valley[0]
valleyradii = numpy.array(valley, dtype=numpy.float64)*self.trimfreq
### do the elliptical average
if self.ellipratio is None:
return None
pixelrdata, rotdata = ctftools.ellipticalAverage(zdata2d, self.ellipratio, self.angle,
self.ringwidth, firstpeak, full=False)
raddata = pixelrdata*self.trimfreq
if self.debug is True:
print "Elliptical CTF limits %.1f A -->> %.1fA"%(1./raddata.min(), 1./raddata.max())
apDisplay.printMsg("Determine and subtract noise model")
CtfNoise = ctfnoise.CtfNoise()
###
### PART 2: BACKGROUND NOISE SUBTRACTION
###
apDisplay.printColor("PART 2: BACKGROUND NOISE SUBTRACTION", "magenta")
### split the function up in first 3/5 and last 3/5 of data with 1/5 overlap
firstvalleyindex = numpy.searchsorted(raddata, self.trimfreq*firstvalley)
numpoints = len(raddata) - firstvalleyindex
# require at least 10 points past first peak of CTF to perform estimation
if numpoints < 10:
apDisplay.printWarning("Not enough points past first peak (n=%d < 10) to do background subtraction"
%(numpoints))
return None
npart1start = firstvalleyindex
npart1end = int(firstvalleyindex + numpoints*6/10.)
npart2start = int(firstvalleyindex + numpoints*5/10.)
npart2end = int(firstvalleyindex + numpoints*9/10.)
npart3start = int(firstvalleyindex + numpoints*8/10.)
npart3end = len(raddata)
svalleydata = ctfnoise.peakExtender(raddata, rotdata, valleyradii, "below")
### fit function below log(CTF), i.e., noise model
## first part data
noisefitparams1 = CtfNoise.modelCTFNoise(raddata[npart1start:npart1end],
svalleydata[npart1start:npart1end], "below")
noisedata1 = CtfNoise.noiseModel(noisefitparams1, raddata)
## second part data
noisefitparams2 = CtfNoise.modelCTFNoise(raddata[npart2start:npart2end],
rotdata[npart2start:npart2end], "below")
noisedata2 = CtfNoise.noiseModel(noisefitparams2, raddata)
## third part data
#noisefitparams3 = CtfNoise.modelCTFNoise(raddata[npart3start:npart3end],
# svalleydata[npart3start:npart3end], "below")
noisefitparams3 = CtfNoise.modelCTFNoise(raddata[npart3start:npart3end],
rotdata[npart3start:npart3end], "below")
noisedata3 = CtfNoise.noiseModel(noisefitparams3, raddata)
## debug only
if self.debug is True:
singlenoisefitparams = CtfNoise.modelCTFNoise(raddata[npart1start:npart3end],
svalleydata[npart1start:npart3end], "below")
singlenoisedata = CtfNoise.noiseModel(singlenoisefitparams, raddata)
## merge data
scale = numpy.arange(npart1end-npart2start, dtype=numpy.float32)
scale /= scale.max()
overlapdata1 = noisedata1[npart2start:npart1end]*(1-scale) + noisedata2[npart2start:npart1end]*scale
scale = numpy.arange(npart2end-npart3start, dtype=numpy.float32)
scale /= scale.max()
overlapdata2 = noisedata2[npart3start:npart2end]*(1-scale) + noisedata3[npart3start:npart2end]*scale
mergedata = numpy.hstack((noisedata1[:npart2start], overlapdata1,
noisedata2[npart1end:npart3start], overlapdata2,
noisedata3[npart2end:]))
noisedata = mergedata
### DO THE SUBTRACTION
normexprotdata = numpy.exp(rotdata) - numpy.exp(noisedata)
### CUT OUT ANY NEGATIVE VALUES FOR DISPLAY AND FITTING PURPOSES ONLY
minval = -1
mindata = ndimage.maximum_filter(normexprotdata, 2)
count = 0
while minval < 3 and count < 10:
count += 1
mindata = ndimage.maximum_filter(mindata, 2)
minval = mindata.min()
if self.debug is True:
apDisplay.printMsg("Minimum value for normalization: %.3f"%(minval))
if minval < 3:
minval = 3
normlogrotdata = numpy.log(numpy.where(normexprotdata<minval, minval, normexprotdata))
if numpy.isnan(normlogrotdata).any() is True:
apDisplay.printError("Error in log normalization of CTF data")
###
### PART 3: ENVELOPE NORMALIZATION
###
apDisplay.printColor("PART 3: ENVELOPE NORMALIZATION", "magenta")
### split the function up in first 3/5 and last 3/5 of data with 1/5 overlap
firstpeakindex = numpy.searchsorted(raddata, firstpeak*self.trimfreq)
numpoints = len(raddata) - firstpeakindex
epart1start = firstpeakindex
epart1end = int(firstpeakindex + numpoints*6/10.)
epart2start = int(firstpeakindex + numpoints*5/10.)
epart2end = int(firstpeakindex + numpoints*9/10.)
epart3start = int(firstpeakindex + numpoints*8/10.)
epart3end = len(raddata)
peakdata = ctfnoise.peakExtender(raddata, normlogrotdata, peakradii, "above")
## first part data
envelopfitparams1 = CtfNoise.modelCTFNoise(raddata[epart1start:epart1end],
peakdata[epart1start:epart1end], "above")
envelopdata1 = CtfNoise.noiseModel(envelopfitparams1, raddata)
## second part data
envelopfitparams2 = CtfNoise.modelCTFNoise(raddata[epart2start:epart2end],
peakdata[epart2start:epart2end], "above")
envelopdata2 = CtfNoise.noiseModel(envelopfitparams2, raddata)
## third part data
envelopfitparams3 = CtfNoise.modelCTFNoise(raddata[epart3start:epart3end],
peakdata[epart3start:epart3end], "above")
envelopdata3 = CtfNoise.noiseModel(envelopfitparams3, raddata)
## merge data
scale = numpy.arange(epart1end-epart2start, dtype=numpy.float32)
scale /= scale.max()
overlapdata1 = envelopdata1[epart2start:epart1end]*(1-scale) + envelopdata2[epart2start:epart1end]*scale
scale = numpy.arange(epart2end-epart3start, dtype=numpy.float32)
scale /= scale.max()
overlapdata2 = envelopdata2[epart3start:epart2end]*(1-scale) + envelopdata3[epart3start:epart2end]*scale
mergedata = numpy.hstack((envelopdata1[:epart2start], overlapdata1,
envelopdata2[epart1end:epart3start], overlapdata2,
envelopdata3[epart2end:]))
envelopdata = mergedata
normnormexprotdata = normexprotdata / numpy.exp(envelopdata)
###
### PART 4: PEAK EXTENSION
###
apDisplay.printColor("PART 4: PEAK EXTENSION", "magenta")
### Subtract fit valley locations
valleydata = ctfnoise.peakExtender(raddata, normnormexprotdata, valleyradii, "below")
valleydata = ndimage.gaussian_filter1d(valleydata, 1)
normvalleydata = normnormexprotdata - valleydata
### Normalize fit peak locations
peakdata = ctfnoise.peakExtender(raddata, normvalleydata, peakradii, "above")
peakdata = ndimage.gaussian_filter1d(peakdata, 1)
normpeakdata = normvalleydata / peakdata
###
### PART 5: CTF FIT AND CONFIDENCE
###
apDisplay.printColor("PART 5: CTF FIT AND CONFIDENCE", "magenta")
### everything in mks units, because rdata is 1/A multiply be 1e10 to get 1/m
ctffitdata = genctf.generateCTF1d(raddata*1e10, focus=meandefocus, cs=self.cs,
volts=self.volts, ampconst=self.ampcontrast, failParams=False)
#ctffitdata2 = genctf.generateCTF1dACE2(raddata*1e10, focus=meandefocus, cs=self.cs,
# volts=self.volts, ampconst=self.ampcontrast, failParams=False)
overctffitdata = genctf.generateCTF1d(raddata*1e10, focus=meandefocus, cs=self.cs,
volts=self.volts, ampconst=self.ampcontrast, failParams=False, overfocus=True)
ind30 = numpy.searchsorted(raddata, 1/30.)
ind10 = numpy.searchsorted(raddata, 1/10.)
self.conf3010 = scipy.stats.pearsonr(normpeakdata[ind30:ind10], ctffitdata[ind30:ind10])[0]
self.overconf3010 = scipy.stats.pearsonr(normpeakdata[ind30:ind10], overctffitdata[ind30:ind10])[0]
apDisplay.printColor("1/30A - 1/10A confidence is %.3f (overfocus %.3f)"%(self.conf3010, self.overconf3010), "green")
if self.overconf3010 > self.conf3010*1.1:
apDisplay.printWarning("Image is possibly over-focused")
ind5peak1 = numpy.searchsorted(raddata, peakradii[0])
ind5peak2 = numpy.searchsorted(raddata, peakradii[5])
self.conf5peak = scipy.stats.pearsonr(normpeakdata[ind5peak1:ind5peak2], ctffitdata[ind5peak1:ind5peak2])[0]
self.overconf5peak = scipy.stats.pearsonr(normpeakdata[ind5peak1:ind5peak2], overctffitdata[ind5peak1:ind5peak2])[0]
apDisplay.printColor("5 peak confidence is %.3f (overfocus %.3f)"%(self.conf5peak, self.overconf5peak), "green")
if self.overconf5peak > self.conf5peak*1.1:
apDisplay.printWarning("Image is possibly over-focused")
###
### PART 6: CTF RESOLUTION LIMITS
###
apDisplay.printColor("PART 6: CTF RESOLUTION LIMITS", "magenta")
confraddata, confdata = ctfres.getCorrelationProfile(raddata,
normpeakdata, ctffitdata, peak, self.trimfreq)
overconfraddata, overconfdata = ctfres.getCorrelationProfile(raddata,
normpeakdata, overctffitdata, peak, self.trimfreq)
self.res80 = ctfres.getResolutionFromConf(confraddata, confdata, limit=0.8)
if self.res80 is None:
self.res80 = 100.0
self.overres80 = ctfres.getResolutionFromConf(overconfraddata, overconfdata, limit=0.8)
if self.overres80 is None:
self.overres80 = 100.0
self.res50 = ctfres.getResolutionFromConf(confraddata, confdata, limit=0.5)
if self.res50 is None:
self.res50 = 100.0
res50max = min(raddata.max(), 1/10.)
elif self.res50 > 15.0:
res50max = min(raddata.max(), 1/10.)
else:
res50max = min(raddata.max(), 1.5/self.res50)
self.overres50 = ctfres.getResolutionFromConf(overconfraddata, overconfdata, limit=0.5)
if self.overres50 is None:
self.overres50 = 100.0
apDisplay.printColor("Resolution limit is %.2f at 0.8 and %.2f at 0.5"
%(self.res80, self.res50), "green")
###
### PART 7: MAKE 1D PLOT SUMMARY FIGURE
###
apDisplay.printColor("PART 7: MAKE 1D PLOT SUMMARY FIGURE", "magenta")
titlefontsize=8
axisfontsize=7
raddatasq = raddata**2
confraddatasq = confraddata**2
valleyradiisq = valleyradii**2
peakradiisq = peakradii**2
fpi = firstpeakindex
pyplot.clf()
if 'subplot2grid' in dir(pyplot):
pyplot.subplot2grid((3,2), (0,0))
else:
pyplot.subplot(2,2,1) # 2 rows, 2 columns, plot 1
pyplot.title("Background Noise Subtraction", fontsize=titlefontsize)
pyplot.ylabel("Log(PSD)", fontsize=axisfontsize)
pyplot.plot(raddata[fpi:], rotdata[fpi:],
'-', color="blue", alpha=0.5, linewidth=0.5)
pyplot.plot(raddata[fpi:], rotdata[fpi:],
'.', color="blue", alpha=0.75, markersize=2.0)
pyplot.plot(raddata[npart1start:npart1end], noisedata1[npart1start:npart1end],
'-', color="magenta", alpha=0.5, linewidth=2)
pyplot.plot(raddata[npart2start:npart2end], noisedata2[npart2start:npart2end],
'-', color="red", alpha=0.5, linewidth=2)
pyplot.plot(raddata[npart3start:npart3end], noisedata3[npart3start:npart3end],
'-', color="orange", alpha=0.5, linewidth=2)
pyplot.plot(raddata[fpi:], noisedata[fpi:],
'--', color="purple", alpha=1.0, linewidth=1)
self.setPyPlotXLabels(raddata, valleyradii=valleyradii, maxloc=res50max)
pyplot.ylim(ymin=noisedata.min())
if 'subplot2grid' in dir(pyplot):
pyplot.subplot2grid((3,2), (0,1))
else:
pyplot.subplot(2,2,2) # 2 rows, 2 columns, plot 2
pyplot.title("Envelope Normalization", fontsize=titlefontsize)
pyplot.ylabel("Log(PSD-Noise)", fontsize=axisfontsize)
pyplot.plot(raddata[fpi:], normlogrotdata[fpi:],
'-', color="blue", alpha=0.5, linewidth=0.5)
pyplot.plot(raddata[fpi:], normlogrotdata[fpi:],
'.', color="blue", alpha=0.75, markersize=2.0)
pyplot.plot(raddata[epart1start:epart1end], envelopdata1[epart1start:epart1end],
'-', color="magenta", alpha=0.5, linewidth=2)
pyplot.plot(raddata[epart2start:epart2end], envelopdata2[epart2start:epart2end],
'-', color="red", alpha=0.5, linewidth=2)
pyplot.plot(raddata[epart3start:epart3end], envelopdata3[epart3start:epart3end],
'-', color="orange", alpha=0.5, linewidth=2)
pyplot.plot(raddata[fpi:], envelopdata[fpi:],
'--', color="purple", alpha=1.0, linewidth=1)
self.setPyPlotXLabels(raddata, peakradii=peakradii, maxloc=res50max)
pyplot.ylim(ymax=envelopdata.max())
if 'subplot2grid' in dir(pyplot):
pyplot.subplot2grid((3,2), (1,0), colspan=2)
else:
pyplot.subplot(2,2,3) # 2 rows, 2 columns, plot 3
pyplot.title("Fit of CTF data (30-10A %.3f / 5-peak %.3f) Def1= %.3e / Def2= %.3e"
%(self.conf3010, self.conf5peak, self.defocus1, self.defocus2), fontsize=titlefontsize)
pyplot.ylabel("Norm PSD", fontsize=titlefontsize)
pyplot.plot(raddatasq[fpi:], ctffitdata[fpi:],
'-', color="black", alpha=0.5, linewidth=1)
#pyplot.plot(raddatasq[fpi:], overctffitdata[fpi:],
# '-', color="red", alpha=0.75, linewidth=1)
pyplot.plot(raddatasq[fpi:], normpeakdata[fpi:],
'-', color="blue", alpha=0.5, linewidth=0.5)
pyplot.plot(raddatasq[fpi:], normpeakdata[fpi:],
'.', color="blue", alpha=0.75, markersize=2.0)
self.setPyPlotXLabels(raddatasq, maxloc=1.0/self.outerAngstrom1D**2, square=True)
pyplot.grid(True, linestyle=':', )
pyplot.ylim(-0.05, 1.05)
"""
pyplot.subplot2grid((3,2), (1,1))
tenangindex = numpy.searchsorted(raddata, 1/10.)-1
pyplot.title("Defocus1= %.3e / Defocus2= %.3e"
%(self.defocus1, self.defocus2), fontsize=titlefontsize)
pyplot.ylabel("Norm PSD", fontsize=titlefontsize)
pyplot.plot(raddatasq[tenangindex:], ctffitdata[tenangindex:],
'-', color="black", alpha=0.5, linewidth=1)
pyplot.plot(raddatasq[tenangindex:], normpeakdata[tenangindex:],
'-', color="blue", alpha=0.5, linewidth=0.5)
pyplot.plot(raddatasq[tenangindex:], normpeakdata[tenangindex:],
'.', color="blue", alpha=0.75, markersize=2.0)
self.setPyPlotXLabels(raddatasq[tenangindex:], maxloc=1/7.**2, square=True)
pyplot.grid(True, linestyle=':', )
pyplot.ylim(-0.05, 1.05)
"""
if 'subplot2grid' in dir(pyplot):
pyplot.subplot2grid((3,2), (2,0), colspan=2)
else:
pyplot.subplot(2,2,4) # 2 rows, 2 columns, plot 4
pyplot.title("Resolution limits: %.2fA at 0.8 and %.2fA at 0.5"
%(self.res80, self.res50), fontsize=titlefontsize)
pyplot.ylabel("Correlation", fontsize=titlefontsize)
pyplot.plot(raddata[fpi:], ctffitdata[fpi:],
'-', color="black", alpha=0.2, linewidth=1)
pyplot.plot(raddata[fpi:], normpeakdata[fpi:],
'-', color="blue", alpha=0.2, linewidth=1)
#pyplot.plot(raddata[fpi:], normpeakdata[fpi:],
# '.', color="black", alpha=0.25, markersize=1.0)
pyplot.axvline(x=1.0/self.res80, linewidth=2, color="gold", alpha=0.95, ymin=0, ymax=0.8)
pyplot.axvline(x=1.0/self.res50, linewidth=2, color="red", alpha=0.95, ymin=0, ymax=0.5)
res80index = numpy.searchsorted(confraddata, 1.0/self.res80)
pyplot.plot(confraddata[:res80index+1], confdata[:res80index+1],
'-', color="green", alpha=1, linewidth=2)
res50index = numpy.searchsorted(confraddata, 1.0/self.res50)
pyplot.plot(confraddata[res80index-1:res50index+1], confdata[res80index-1:res50index+1],
'-', color="orange", alpha=1, linewidth=2)
pyplot.plot(confraddata[res50index-1:], confdata[res50index-1:],
'-', color="red", alpha=1, linewidth=2)
self.setPyPlotXLabels(raddata, maxloc=res50max)
pyplot.grid(True, linestyle=':', )
if self.res80 < 99:
pyplot.ylim(-0.05, 1.05)
elif self.res50 < 99:
pyplot.ylim(-0.25, 1.05)
else:
pyplot.ylim(-0.55, 1.05)
pyplot.subplots_adjust(wspace=0.22, hspace=0.50,
bottom=0.08, left=0.07, top=0.95, right=0.965, )
self.plotsfile = apDisplay.short(self.imgname)+"-plots.png"
apDisplay.printMsg("Saving 1D graph to file %s"%(self.plotsfile))
pyplot.savefig(self.plotsfile, format="png", dpi=300, orientation='landscape', pad_inches=0.0)
if self.debug is True:
### write a 1d profile dat files
f = open(apDisplay.short(self.imgname)+"-noise_fit.dat", "w")
for i in range(npart1start, npart3end):
f.write("%.16f\t%.16f\t%.16f\t%.16f\n"%(raddata[i], rotdata[i], singlenoisedata[i], noisedata[i]))
f.write("&\n")
for i in range(npart1start, npart1end):
f.write("%.16f\t%.16f\n"%(raddata[i], noisedata1[i]))
f.write("&\n")
for i in range(npart2start, npart2end):
f.write("%.16f\t%.16f\n"%(raddata[i], noisedata2[i]))
f.write("&\n")
for i in range(npart3start, npart3end):
f.write("%.16f\t%.16f\n"%(raddata[i], noisedata3[i]))
f.write("&\n")
f.close()
#smallrotdata = numpy.where(rotdata-singlenoisedata>0.19, 0.19, rotdata-singlenoisedata)
noiseexp = numpy.exp(singlenoisedata)
smallrotdata = numpy.exp(rotdata) - noiseexp
minval = 3
smallrotdata = numpy.log(numpy.where(smallrotdata<minval, minval, smallrotdata))
smallnoise = numpy.exp(noisedata) - noiseexp
smallnoise = numpy.log(numpy.where(smallnoise<minval, minval, smallnoise))
smallnoise1 = numpy.exp(noisedata1) - noiseexp
smallnoise1 = numpy.log(numpy.where(smallnoise1<minval, minval, smallnoise1))
smallnoise2 = numpy.exp(noisedata2) - noiseexp
smallnoise2 = numpy.log(numpy.where(smallnoise2<minval, minval, smallnoise2))
smallnoise3 = numpy.exp(noisedata3) - noiseexp
smallnoise3 = numpy.log(numpy.where(smallnoise3<minval, minval, smallnoise3))
f = open(apDisplay.short(self.imgname)+"-noisesubt_fit.dat", "w")
for i in range(len(ctffitdata)):
f.write("%.16f\t%.16f\n"%(raddata[i], smallrotdata[i]))
f.write("&\n")
for i in range(npart1start, npart3end):
f.write("%.16f\t%.16f\t%.16f\t%.16f\n"%(raddata[i], smallrotdata[i], smallnoise[i], 0))
f.write("&\n")
for i in range(npart1start, npart1end):
f.write("%.16f\t%.16f\n"%(raddata[i], smallnoise1[i]))
f.write("&\n")
for i in range(npart2start, npart2end):
f.write("%.16f\t%.16f\n"%(raddata[i], smallnoise2[i]))
f.write("&\n")
for i in range(npart3start, npart3end):
f.write("%.16f\t%.16f\n"%(raddata[i], smallnoise3[i]))
f.write("&\n")
f.close()
f = open(apDisplay.short(self.imgname)+"-ctf_fit.dat", "w")
for i in range(len(ctffitdata)):
f.write("%.16f\t%.16f\t%.16f\n"%(raddata[i], normpeakdata[i], ctffitdata[i]))
f.close()
#sys.exit(1)
if self.debug is True:
print "Showing results"
#pyplot.show()
#plotspng = Image.open(self.plotsfile)
#plotspng.show()
pyplot.clf()
if twod is False:
return zdata2d
###
### PART 8: NORMALIZE THE 2D IMAGE
###
apDisplay.printColor("PART 8: NORMALIZE THE 2D IMAGE", "magenta")
### Convert 1D array into 2D array by un-elliptical average
noise2d = ctftools.unEllipticalAverage(pixelrdata, noisedata,
self.ellipratio, self.angle, zdata2d.shape)
envelop2d = ctftools.unEllipticalAverage(pixelrdata, envelopdata,
self.ellipratio, self.angle, zdata2d.shape)
valley2d = ctftools.unEllipticalAverage(pixelrdata, valleydata,
self.ellipratio, self.angle, zdata2d.shape)
peak2d = ctftools.unEllipticalAverage(pixelrdata, peakdata,
self.ellipratio, self.angle, zdata2d.shape)
### Do the normalization on the 2d data
#blur2d = ndimage.gaussian_filter(zdata2d, 2)
normal2d = numpy.exp(zdata2d) - numpy.exp(noise2d)
normal2d = normal2d / numpy.exp(envelop2d)
normal2d = normal2d - valley2d
normal2d = normal2d / peak2d
normal2d = numpy.where(normal2d < -0.2, -0.2, normal2d)
normal2d = numpy.where(normal2d > 1.2, 1.2, normal2d)
return normal2d