def chebyshev_fit(self, x_obs, y_obs, w_obs, n_terms=None): from scitbx.math import chebyshev_polynome from scitbx.math import chebyshev_lsq_fit if n_terms is None: # determining the number of terms takes much, much longer than the fit n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=5, max_terms=20, n_goes=20, n_free=20) self.logger.info("Fitting with %i terms" % n_terms) fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs) self.logger.info("Least Squares residual: %7.6f" % (fit.f)) fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x_obs) if 0: # debugging plots from matplotlib import pyplot pyplot.clf() pyplot.plot(x_obs, y_obs) pyplot.plot(x_obs, y_fitted) pyplot.draw() pyplot.show() return y_fitted
def estimate_signal_to_noise(x, y): raise if 1: x, y = interpolate(x, y) #x, y_tr = fourier_filter(x, y) x, y_tr = savitzky_golay_filter(x, y) noise = y - y_tr else: from scitbx.math import chebyshev_polynome from scitbx.math import chebyshev_lsq_fit x_obs, y_obs = x, y w_obs = flex.double(y_obs.size(), 1) w_obs[0] = 1e16 w_obs[-1] = 1e16 ## determining the number of terms takes much, much longer than the fit n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=2, max_terms=30, n_goes=20, n_free=20) #n_terms = 7 print "n_terms:", n_terms fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs) fit_funct = chebyshev_polynome( n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x) y_tr = y_fitted n = y_tr.size() noise = y - y_tr noise_sq = flex.pow2(noise) from xfel.command_line.view_pixel_histograms import sliding_average #sigma_sq = sliding_average(noise_sq, n=31) sigma_sq = sliding_average(noise_sq, n=15) #sigma_sq = sliding_average(sigma_sq) #signal_to_noise = y/flex.sqrt(sigma_sq) import math signal_to_noise = y/math.sqrt(flex.mean(noise_sq[50:200])) #pyplot.plot(noise) #pyplot.plot(x,y) #pyplot.show() offset = 0.2 * flex.max(y) offset = 0 pyplot.plot(x, y, linewidth=2) pyplot.plot(x, offset+y_tr, linewidth=2) pyplot.show() pyplot.plot(x, noise, linewidth=2) #pyplot.plot(x, flex.sqrt(sigma_sq), linewidth=2) #ax2 = pyplot.twinx() #ax2.plot(x, y) pyplot.show() pyplot.plot(x[:375], signal_to_noise[:375]) #pyplot.xlim( #ax2 = pyplot.twinx() #ax2.plot(x, y) pyplot.show()
def chebyshev_fit(self, x_obs, y_obs, w_obs, n_terms=None): from scitbx.math import chebyshev_polynome from scitbx.math import chebyshev_lsq_fit if n_terms is None: # determining the number of terms takes much, much longer than the fit n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=5, max_terms=20, n_goes=20, n_free=20) self.logger.info("Fitting with %i terms" %n_terms) fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs) self.logger.info("Least Squares residual: %7.6f" %(fit.f)) fit_funct = chebyshev_polynome( n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x_obs) if 0: # debugging plots from matplotlib import pyplot pyplot.clf() pyplot.plot(x_obs, y_obs) pyplot.plot(x_obs, y_fitted) pyplot.draw() pyplot.show() return y_fitted
def estimate_signal_to_noise(x, y): raise if 1: x, y = interpolate(x, y) #x, y_tr = fourier_filter(x, y) x, y_tr = savitzky_golay_filter(x, y) noise = y - y_tr else: from scitbx.math import chebyshev_polynome from scitbx.math import chebyshev_lsq_fit x_obs, y_obs = x, y w_obs = flex.double(y_obs.size(), 1) w_obs[0] = 1e16 w_obs[-1] = 1e16 ## determining the number of terms takes much, much longer than the fit n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=2, max_terms=30, n_goes=20, n_free=20) #n_terms = 7 print "n_terms:", n_terms fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs) fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x) y_tr = y_fitted n = y_tr.size() noise = y - y_tr noise_sq = flex.pow2(noise) from xfel.command_line.view_pixel_histograms import sliding_average #sigma_sq = sliding_average(noise_sq, n=31) sigma_sq = sliding_average(noise_sq, n=15) #sigma_sq = sliding_average(sigma_sq) #signal_to_noise = y/flex.sqrt(sigma_sq) import math signal_to_noise = y / math.sqrt(flex.mean(noise_sq[50:200])) #pyplot.plot(noise) #pyplot.plot(x,y) #pyplot.show() offset = 0.2 * flex.max(y) offset = 0 pyplot.plot(x, y, linewidth=2) pyplot.plot(x, offset + y_tr, linewidth=2) pyplot.show() pyplot.plot(x, noise, linewidth=2) #pyplot.plot(x, flex.sqrt(sigma_sq), linewidth=2) #ax2 = pyplot.twinx() #ax2.plot(x, y) pyplot.show() pyplot.plot(x[:375], signal_to_noise[:375]) #pyplot.xlim( #ax2 = pyplot.twinx() #ax2.plot(x, y) pyplot.show()
def example(): x_obs = (flex.double(range(100)) + 1.0) / 101.0 y_ideal = flex.sin(x_obs * 6.0 * 3.1415) + flex.exp(x_obs) y_obs = y_ideal + (flex.random_double(size=x_obs.size()) - 0.5) * 0.5 w_obs = flex.double(x_obs.size(), 1) print "Trying to determine the best number of terms " print " via cross validation techniques" print n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=5, max_terms=20, n_goes=20, n_free=20) print "Fitting with", n_terms, "terms" print fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs) print "Least Squares residual: %7.6f" % (fit.f) print " R2-value : %7.6f" % (fit.f / flex.sum(y_obs * y_obs)) print fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x_obs) abs_deviation = flex.max(flex.abs((y_ideal - y_fitted))) print "Maximum deviation between fitted and error free data:" print " %4.3f" % (abs_deviation) abs_deviation = flex.mean(flex.abs((y_ideal - y_fitted))) print "Mean deviation between fitted and error free data:" print " %4.3f" % (abs_deviation) print abs_deviation = flex.max(flex.abs((y_obs - y_fitted))) print "Maximum deviation between fitted and observed data:" print " %4.3f" % (abs_deviation) abs_deviation = flex.mean(flex.abs((y_obs - y_fitted))) print "Mean deviation between fitted and observed data:" print " %4.3f" % (abs_deviation) print print "Showing 10 points" print " x y_obs y_ideal y_fit" for ii in range(10): print "%6.3f %6.3f %6.3f %6.3f" \ %(x_obs[ii*9], y_obs[ii*9], y_ideal[ii*9], y_fitted[ii*9]) try: from iotbx import data_plots except ImportError: pass else: print "Preparing output for loggraph in a file called" print " chebyshev.loggraph" chebyshev_plot = data_plots.plot_data(plot_title='Chebyshev fitting', x_label='x values', y_label='y values', x_data=x_obs, y_data=y_obs, y_legend='Observed y values', comments='Chebyshev fit') chebyshev_plot.add_data(y_data=y_ideal, y_legend='Error free y values') chebyshev_plot.add_data(y_data=y_fitted, y_legend='Fitted chebyshev approximation') output_logfile = open('chebyshev.loggraph', 'w') f = StringIO() data_plots.plot_data_loggraph(chebyshev_plot, f) output_logfile.write(f.getvalue())
def example(): x_obs = (flex.double(range(100))+1.0)/101.0 y_ideal = flex.sin(x_obs*6.0*3.1415) + flex.exp(x_obs) y_obs = y_ideal + (flex.random_double(size=x_obs.size())-0.5)*0.5 w_obs = flex.double(x_obs.size(),1) print "Trying to determine the best number of terms " print " via cross validation techniques" print n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs,y_obs,w_obs, min_terms=5 ,max_terms=20, n_goes=20,n_free=20) print "Fitting with", n_terms, "terms" print fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms,x_obs,y_obs) print "Least Squares residual: %7.6f" %(fit.f) print " R2-value : %7.6f" %(fit.f/flex.sum(y_obs*y_obs)) print fit_funct = chebyshev_polynome( n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x_obs) abs_deviation = flex.max( flex.abs( (y_ideal- y_fitted) ) ) print "Maximum deviation between fitted and error free data:" print " %4.3f" %(abs_deviation) abs_deviation = flex.mean( flex.abs( (y_ideal- y_fitted) ) ) print "Mean deviation between fitted and error free data:" print " %4.3f" %(abs_deviation) print abs_deviation = flex.max( flex.abs( (y_obs- y_fitted) ) ) print "Maximum deviation between fitted and observed data:" print " %4.3f" %(abs_deviation) abs_deviation = flex.mean( flex.abs( (y_obs- y_fitted) ) ) print "Mean deviation between fitted and observed data:" print " %4.3f" %(abs_deviation) print print "Showing 10 points" print " x y_obs y_ideal y_fit" for ii in range(10): print "%6.3f %6.3f %6.3f %6.3f" \ %(x_obs[ii*9], y_obs[ii*9], y_ideal[ii*9], y_fitted[ii*9]) try: from iotbx import data_plots except ImportError: pass else: print "Preparing output for loggraph in a file called" print " chebyshev.loggraph" chebyshev_plot = data_plots.plot_data(plot_title='Chebyshev fitting', x_label = 'x values', y_label = 'y values', x_data = x_obs, y_data = y_obs, y_legend = 'Observed y values', comments = 'Chebyshev fit') chebyshev_plot.add_data(y_data=y_ideal, y_legend='Error free y values') chebyshev_plot.add_data(y_data=y_fitted, y_legend='Fitted chebyshev approximation') output_logfile=open('chebyshev.loggraph','w') f = StringIO() data_plots.plot_data_loggraph(chebyshev_plot,f) output_logfile.write(f.getvalue())