def chebyshev_fit(self, x_obs, y_obs, w_obs, n_terms=None):
from scitbx.math import chebyshev_polynome
from scitbx.math import chebyshev_lsq_fit
if n_terms is None:
# determining the number of terms takes much, much longer than the fit
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs,
y_obs,
w_obs,
min_terms=5,
max_terms=20,
n_goes=20,
n_free=20)
self.logger.info("Fitting with %i terms" % n_terms)
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs)
self.logger.info("Least Squares residual: %7.6f" % (fit.f))
fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit,
fit.coefs)
y_fitted = fit_funct.f(x_obs)
if 0:
# debugging plots
from matplotlib import pyplot
pyplot.clf()
pyplot.plot(x_obs, y_obs)
pyplot.plot(x_obs, y_fitted)
pyplot.draw()
pyplot.show()
return y_fitted
def estimate_signal_to_noise(x, y):
raise
if 1:
x, y = interpolate(x, y)
#x, y_tr = fourier_filter(x, y)
x, y_tr = savitzky_golay_filter(x, y)
noise = y - y_tr
else:
from scitbx.math import chebyshev_polynome
from scitbx.math import chebyshev_lsq_fit
x_obs, y_obs = x, y
w_obs = flex.double(y_obs.size(), 1)
w_obs[0] = 1e16
w_obs[-1] = 1e16
## determining the number of terms takes much, much longer than the fit
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs, y_obs, w_obs,
min_terms=2, max_terms=30,
n_goes=20, n_free=20)
#n_terms = 7
print "n_terms:", n_terms
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs)
fit_funct = chebyshev_polynome(
n_terms, fit.low_limit, fit.high_limit, fit.coefs)
y_fitted = fit_funct.f(x)
y_tr = y_fitted
n = y_tr.size()
noise = y - y_tr
noise_sq = flex.pow2(noise)
from xfel.command_line.view_pixel_histograms import sliding_average
#sigma_sq = sliding_average(noise_sq, n=31)
sigma_sq = sliding_average(noise_sq, n=15)
#sigma_sq = sliding_average(sigma_sq)
#signal_to_noise = y/flex.sqrt(sigma_sq)
import math
signal_to_noise = y/math.sqrt(flex.mean(noise_sq[50:200]))
#pyplot.plot(noise)
#pyplot.plot(x,y)
#pyplot.show()
offset = 0.2 * flex.max(y)
offset = 0
pyplot.plot(x, y, linewidth=2)
pyplot.plot(x, offset+y_tr, linewidth=2)
pyplot.show()
pyplot.plot(x, noise, linewidth=2)
#pyplot.plot(x, flex.sqrt(sigma_sq), linewidth=2)
#ax2 = pyplot.twinx()
#ax2.plot(x, y)
pyplot.show()
pyplot.plot(x[:375], signal_to_noise[:375])
#pyplot.xlim(
#ax2 = pyplot.twinx()
#ax2.plot(x, y)
pyplot.show()
def chebyshev_fit(self, x_obs, y_obs, w_obs, n_terms=None):
from scitbx.math import chebyshev_polynome
from scitbx.math import chebyshev_lsq_fit
if n_terms is None:
# determining the number of terms takes much, much longer than the fit
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs, y_obs, w_obs,
min_terms=5, max_terms=20,
n_goes=20, n_free=20)
self.logger.info("Fitting with %i terms" %n_terms)
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs)
self.logger.info("Least Squares residual: %7.6f" %(fit.f))
fit_funct = chebyshev_polynome(
n_terms, fit.low_limit, fit.high_limit, fit.coefs)
y_fitted = fit_funct.f(x_obs)
if 0:
# debugging plots
from matplotlib import pyplot
pyplot.clf()
pyplot.plot(x_obs, y_obs)
pyplot.plot(x_obs, y_fitted)
pyplot.draw()
pyplot.show()
return y_fitted
def estimate_signal_to_noise(x, y):
raise
if 1:
x, y = interpolate(x, y)
#x, y_tr = fourier_filter(x, y)
x, y_tr = savitzky_golay_filter(x, y)
noise = y - y_tr
else:
from scitbx.math import chebyshev_polynome
from scitbx.math import chebyshev_lsq_fit
x_obs, y_obs = x, y
w_obs = flex.double(y_obs.size(), 1)
w_obs[0] = 1e16
w_obs[-1] = 1e16
## determining the number of terms takes much, much longer than the fit
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs,
y_obs,
w_obs,
min_terms=2,
max_terms=30,
n_goes=20,
n_free=20)
#n_terms = 7
print "n_terms:", n_terms
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs)
fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit,
fit.coefs)
y_fitted = fit_funct.f(x)
y_tr = y_fitted
n = y_tr.size()
noise = y - y_tr
noise_sq = flex.pow2(noise)
from xfel.command_line.view_pixel_histograms import sliding_average
#sigma_sq = sliding_average(noise_sq, n=31)
sigma_sq = sliding_average(noise_sq, n=15)
#sigma_sq = sliding_average(sigma_sq)
#signal_to_noise = y/flex.sqrt(sigma_sq)
import math
signal_to_noise = y / math.sqrt(flex.mean(noise_sq[50:200]))
#pyplot.plot(noise)
#pyplot.plot(x,y)
#pyplot.show()
offset = 0.2 * flex.max(y)
offset = 0
pyplot.plot(x, y, linewidth=2)
pyplot.plot(x, offset + y_tr, linewidth=2)
pyplot.show()
pyplot.plot(x, noise, linewidth=2)
#pyplot.plot(x, flex.sqrt(sigma_sq), linewidth=2)
#ax2 = pyplot.twinx()
#ax2.plot(x, y)
pyplot.show()
pyplot.plot(x[:375], signal_to_noise[:375])
#pyplot.xlim(
#ax2 = pyplot.twinx()
#ax2.plot(x, y)
pyplot.show()
def example():
x_obs = (flex.double(range(100)) + 1.0) / 101.0
y_ideal = flex.sin(x_obs * 6.0 * 3.1415) + flex.exp(x_obs)
y_obs = y_ideal + (flex.random_double(size=x_obs.size()) - 0.5) * 0.5
w_obs = flex.double(x_obs.size(), 1)
print "Trying to determine the best number of terms "
print " via cross validation techniques"
print
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs, y_obs, w_obs, min_terms=5, max_terms=20, n_goes=20, n_free=20)
print "Fitting with", n_terms, "terms"
print
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs)
print "Least Squares residual: %7.6f" % (fit.f)
print " R2-value : %7.6f" % (fit.f / flex.sum(y_obs * y_obs))
print
fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit,
fit.coefs)
y_fitted = fit_funct.f(x_obs)
abs_deviation = flex.max(flex.abs((y_ideal - y_fitted)))
print "Maximum deviation between fitted and error free data:"
print " %4.3f" % (abs_deviation)
abs_deviation = flex.mean(flex.abs((y_ideal - y_fitted)))
print "Mean deviation between fitted and error free data:"
print " %4.3f" % (abs_deviation)
print
abs_deviation = flex.max(flex.abs((y_obs - y_fitted)))
print "Maximum deviation between fitted and observed data:"
print " %4.3f" % (abs_deviation)
abs_deviation = flex.mean(flex.abs((y_obs - y_fitted)))
print "Mean deviation between fitted and observed data:"
print " %4.3f" % (abs_deviation)
print
print "Showing 10 points"
print " x y_obs y_ideal y_fit"
for ii in range(10):
print "%6.3f %6.3f %6.3f %6.3f" \
%(x_obs[ii*9], y_obs[ii*9], y_ideal[ii*9], y_fitted[ii*9])
try:
from iotbx import data_plots
except ImportError:
pass
else:
print "Preparing output for loggraph in a file called"
print " chebyshev.loggraph"
chebyshev_plot = data_plots.plot_data(plot_title='Chebyshev fitting',
x_label='x values',
y_label='y values',
x_data=x_obs,
y_data=y_obs,
y_legend='Observed y values',
comments='Chebyshev fit')
chebyshev_plot.add_data(y_data=y_ideal, y_legend='Error free y values')
chebyshev_plot.add_data(y_data=y_fitted,
y_legend='Fitted chebyshev approximation')
output_logfile = open('chebyshev.loggraph', 'w')
f = StringIO()
data_plots.plot_data_loggraph(chebyshev_plot, f)
output_logfile.write(f.getvalue())
def example():
x_obs = (flex.double(range(100))+1.0)/101.0
y_ideal = flex.sin(x_obs*6.0*3.1415) + flex.exp(x_obs)
y_obs = y_ideal + (flex.random_double(size=x_obs.size())-0.5)*0.5
w_obs = flex.double(x_obs.size(),1)
print "Trying to determine the best number of terms "
print " via cross validation techniques"
print
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs,y_obs,w_obs,
min_terms=5 ,max_terms=20,
n_goes=20,n_free=20)
print "Fitting with", n_terms, "terms"
print
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms,x_obs,y_obs)
print "Least Squares residual: %7.6f" %(fit.f)
print " R2-value : %7.6f" %(fit.f/flex.sum(y_obs*y_obs))
print
fit_funct = chebyshev_polynome(
n_terms, fit.low_limit, fit.high_limit, fit.coefs)
y_fitted = fit_funct.f(x_obs)
abs_deviation = flex.max(
flex.abs( (y_ideal- y_fitted) ) )
print "Maximum deviation between fitted and error free data:"
print " %4.3f" %(abs_deviation)
abs_deviation = flex.mean(
flex.abs( (y_ideal- y_fitted) ) )
print "Mean deviation between fitted and error free data:"
print " %4.3f" %(abs_deviation)
print
abs_deviation = flex.max(
flex.abs( (y_obs- y_fitted) ) )
print "Maximum deviation between fitted and observed data:"
print " %4.3f" %(abs_deviation)
abs_deviation = flex.mean(
flex.abs( (y_obs- y_fitted) ) )
print "Mean deviation between fitted and observed data:"
print " %4.3f" %(abs_deviation)
print
print "Showing 10 points"
print " x y_obs y_ideal y_fit"
for ii in range(10):
print "%6.3f %6.3f %6.3f %6.3f" \
%(x_obs[ii*9], y_obs[ii*9], y_ideal[ii*9], y_fitted[ii*9])
try:
from iotbx import data_plots
except ImportError:
pass
else:
print "Preparing output for loggraph in a file called"
print " chebyshev.loggraph"
chebyshev_plot = data_plots.plot_data(plot_title='Chebyshev fitting',
x_label = 'x values',
y_label = 'y values',
x_data = x_obs,
y_data = y_obs,
y_legend = 'Observed y values',
comments = 'Chebyshev fit')
chebyshev_plot.add_data(y_data=y_ideal,
y_legend='Error free y values')
chebyshev_plot.add_data(y_data=y_fitted,
y_legend='Fitted chebyshev approximation')
output_logfile=open('chebyshev.loggraph','w')
f = StringIO()
data_plots.plot_data_loggraph(chebyshev_plot,f)
output_logfile.write(f.getvalue())
