def runge_phenomenon(self,n=41,nt=35,print_it=False): x_e = 2.0*(flex.double( xrange(n) )/float(n-1)-0.5) y_e = 1/(1+x_e*x_e*25) fit_e = chebyshev_lsq_fit.chebyshev_lsq_fit(nt, x_e, y_e, ) fit_e = chebyshev_polynome( nt, fit_e.low_limit, fit_e.high_limit, fit_e.coefs) x_c = chebyshev_lsq_fit.chebyshev_nodes(n, -1, 1, True) y_c = 1/(1+x_c*x_c*25) fit_c = chebyshev_lsq_fit.chebyshev_lsq_fit(nt, x_c, y_c, ) fit_c = chebyshev_polynome( nt, fit_c.low_limit, fit_c.high_limit, fit_c.coefs) x_plot = 2.0*(flex.double( xrange(3*n) )/float(3*n-1)-0.5) y_plot_e = fit_e.f( x_plot ) y_plot_c = fit_c.f( x_plot ) y_id = 1/(1+x_plot*x_plot*25) if print_it: for x,y,yy,yyy in zip(x_plot,y_id,y_plot_e,y_plot_c): print x,y,yy,yyy
def another_example(np=41,nt=5): x = flex.double( range(np) )/(np-1) y = 0.99*flex.exp(-x*x*0.5) y = -flex.log(1.0/y-1) w = y*y/1.0 d = (flex.random_double(np)-0.5)*w y_obs = y+d y = 1.0/( 1.0 + flex.exp(-y) ) fit_w = chebyshev_lsq_fit.chebyshev_lsq_fit(nt, x, y_obs, w ) fit_w_f = chebyshev_polynome( nt, fit_w.low_limit, fit_w.high_limit, fit_w.coefs) fit_nw = chebyshev_lsq_fit.chebyshev_lsq_fit(nt, x, y_obs) fit_nw_f = chebyshev_polynome( nt, fit_nw.low_limit, fit_nw.high_limit, fit_nw.coefs) print print "Coefficients from weighted lsq" print list( fit_w.coefs ) print "Coefficients from non-weighted lsq" print list( fit_nw.coefs ) assert flex.max( flex.abs(fit_nw.coefs-fit_w.coefs) ) > 0
def runge_phenomenon(self, n=41, nt=35, print_it=False): x_e = 2.0 * (flex.double(xrange(n)) / float(n - 1) - 0.5) y_e = 1 / (1 + x_e * x_e * 25) fit_e = chebyshev_lsq_fit.chebyshev_lsq_fit( nt, x_e, y_e, ) fit_e = chebyshev_polynome(nt, fit_e.low_limit, fit_e.high_limit, fit_e.coefs) x_c = chebyshev_lsq_fit.chebyshev_nodes(n, -1, 1, True) y_c = 1 / (1 + x_c * x_c * 25) fit_c = chebyshev_lsq_fit.chebyshev_lsq_fit( nt, x_c, y_c, ) fit_c = chebyshev_polynome(nt, fit_c.low_limit, fit_c.high_limit, fit_c.coefs) x_plot = 2.0 * (flex.double(xrange(3 * n)) / float(3 * n - 1) - 0.5) y_plot_e = fit_e.f(x_plot) y_plot_c = fit_c.f(x_plot) y_id = 1 / (1 + x_plot * x_plot * 25) if print_it: for x, y, yy, yyy in zip(x_plot, y_id, y_plot_e, y_plot_c): print x, y, yy, yyy
def __init__(self): self.lims = (0, 3.999) self.n = 25 self.cheb_coefs_mean = flex.double([ -5.7657057160113165, -1.4401697329647773, 0.27024527974695978, 0.0078985123178169463, -0.09631162819805221, 0.10735910911235932, -0.2233791415486541, 0.19498973606201189, -0.26350484909274735, 0.27066922616706718, -0.25090257310775238, 0.22483113451869022, -0.16726514028401993, 0.1027196386216821, -0.091673985006882924, 0.052259467525186447, -0.016970937165037378, 0.031137149395069535, 0.007431391017472774, 0.014012399241218143, 0.010042763559628737, -0.015341614910420142, 0.0068872073662354753, 0.012869250293871155, 0.008959831186295333 ]) self.cheb_coefs_low = flex.double([ -9.2669907863291705, -1.9456068148305121, 0.86779111670403164, 0.14288741649825987, 0.052348627094772476, -0.075972113378021927, -0.24684531148041183, 0.26186454415838456, -0.11901905265242005, 0.51976185959989885, -0.45937166668405666, 0.31225783754328595, -0.16416500741911164, 0.32535788332409726, -0.17264130742234479, 0.049992516172585558, -0.11856771807344196, 0.044221277456559308, 0.18118326484913538, 0.0550906704796576, 0.047995789360044186, -0.083644664646019343, 0.029230535847010842, -0.040766646640578184, -0.070401435043602661 ]) self.cheb_coefs_high = flex.double([ -2.4313071167283367, -1.0123006175290996, -0.15088667146206206, -0.13439270400442058, 0.0096372912886536471, 0.063670920315285234, -0.21889541306029214, -0.0062901907374255505, -0.17570764254872775, 0.0052730332601467964, 0.010615988987238418, 0.14617014871482245, 0.021082968158927021, -0.0046556220561053659, -0.02634693069601177, 0.026402323365458613, -0.051317703298683792, 0.00089074796092045432, 0.011927144561737445, 0.040880441622645626, 0.028415795670163915, -0.01415133080863344, -0.02323296702274262, -5.8561934669618548e-05, 0.051520439604993466 ]) self.loc = 0.10 self.scale = 1.0 self.m = chebyshev_polynome(self.n, self.lims[0] * 0.99999, self.lims[1] / 0.99999, self.cheb_coefs_mean) self.low = chebyshev_polynome(self.n, self.lims[0] * 0.99999, self.lims[1] / 0.99999, self.cheb_coefs_low) self.high = chebyshev_polynome(self.n, self.lims[0] * 0.99999, self.lims[1] / 0.99999, self.cheb_coefs_high)
def __init__(self, x, y, order): self.x = x self.y = y self.n = order coefs = flex.random_double(self.n) self.polynome = chebyshev_polynome(self.n, -1.0, 1.0, coefs) self.optimize()
def chebyshev_fit(self, x_obs, y_obs, w_obs, n_terms=None): from scitbx.math import chebyshev_polynome from scitbx.math import chebyshev_lsq_fit if n_terms is None: # determining the number of terms takes much, much longer than the fit n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=5, max_terms=20, n_goes=20, n_free=20) self.logger.info("Fitting with %i terms" % n_terms) fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs) self.logger.info("Least Squares residual: %7.6f" % (fit.f)) fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x_obs) if 0: # debugging plots from matplotlib import pyplot pyplot.clf() pyplot.plot(x_obs, y_obs) pyplot.plot(x_obs, y_fitted) pyplot.draw() pyplot.show() return y_fitted
def normalize(self): result = self.compute_normalisation_constant() self.coefs[0] = self.coefs[0] - 2.0 * math.log(result) self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs) result = self.compute_normalisation_constant() if abs(result - 1.0) > 1e-6: self.normalize()
def load_coefs(self, coefs=None, random_scale=1.0, thres=600): if coefs is not None: assert len(self.coefs) == self.n self.coefs = coefs else: self.coefs = random_scale * (flex.random_double(self.n) * 2 - 1.0) self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs) self.normalize()
def load_coefs(self, coefs=None): if coefs is None: self.coefs = (flex.random_double(self.n)-0.5)*2.0 else: assert len(coefs)==self.n self.coefs = coefs # no means to refresh the coefficients yet in an elegant manner self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs)
def load_coefs(self, coefs=None): if coefs is None: self.coefs = (flex.random_double(self.n) - 0.5) * 2.0 else: assert len(coefs) == self.n self.coefs = coefs # no means to refresh the coefficients yet in an elegant manner self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs)
def estimate_signal_to_noise(x, y): raise if 1: x, y = interpolate(x, y) #x, y_tr = fourier_filter(x, y) x, y_tr = savitzky_golay_filter(x, y) noise = y - y_tr else: from scitbx.math import chebyshev_polynome from scitbx.math import chebyshev_lsq_fit x_obs, y_obs = x, y w_obs = flex.double(y_obs.size(), 1) w_obs[0] = 1e16 w_obs[-1] = 1e16 ## determining the number of terms takes much, much longer than the fit n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=2, max_terms=30, n_goes=20, n_free=20) #n_terms = 7 print "n_terms:", n_terms fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs) fit_funct = chebyshev_polynome( n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x) y_tr = y_fitted n = y_tr.size() noise = y - y_tr noise_sq = flex.pow2(noise) from xfel.command_line.view_pixel_histograms import sliding_average #sigma_sq = sliding_average(noise_sq, n=31) sigma_sq = sliding_average(noise_sq, n=15) #sigma_sq = sliding_average(sigma_sq) #signal_to_noise = y/flex.sqrt(sigma_sq) import math signal_to_noise = y/math.sqrt(flex.mean(noise_sq[50:200])) #pyplot.plot(noise) #pyplot.plot(x,y) #pyplot.show() offset = 0.2 * flex.max(y) offset = 0 pyplot.plot(x, y, linewidth=2) pyplot.plot(x, offset+y_tr, linewidth=2) pyplot.show() pyplot.plot(x, noise, linewidth=2) #pyplot.plot(x, flex.sqrt(sigma_sq), linewidth=2) #ax2 = pyplot.twinx() #ax2.plot(x, y) pyplot.show() pyplot.plot(x[:375], signal_to_noise[:375]) #pyplot.xlim( #ax2 = pyplot.twinx() #ax2.plot(x, y) pyplot.show()
def calibration(pr_1, pr_2, n_params): x1 = pr_1.r / pr_1.r[-1] x2 = pr_2.r / pr_2.r[-1] cdf_1 = pr_1.pr2cdf() cdf_2 = pr_2.pr2cdf() new_cdf_2 = flex.linear_interpolation(x2, cdf_2, x1).deep_copy() fitting = nonlinear_fit(new_cdf_2, cdf_1, n_params) solution = fitting.solution fn = chebyshev_polynome(n_params, -1.0, 1.0, solution) return fn
def __init__(self, n, m=100, k=2.5, d_max=45.0): self.n = n self.m = m self.k = k self.d_max=d_max self.x = 1.0-2.0*(flex.double(range(m+1))/m) self.r = 0.5*(1+self.x)*self.d_max self.r[0] = 1e-8 self.coefs = (flex.random_double(self.n)-0.5)*0.0 self.load_coefs() self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs)
def __init__(self, n, m=100, k=2.5, d_max=45.0): self.n = n self.m = m self.k = k self.d_max = d_max self.x = 1.0 - 2.0 * (flex.double(range(m + 1)) / m) self.r = 0.5 * (1 + self.x) * self.d_max self.r[0] = 1e-8 self.coefs = (flex.random_double(self.n) - 0.5) * 0.0 self.load_coefs() self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs)
def __init__(self, prior, n, coefs=None, n_int=25): self.prior = prior self.coefs = coefs self.n = n self.base_name = "pofx" if self.coefs is None: self.coefs = flex.double([0] * self.n) self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs) gle = scitbx.math.gauss_legendre_engine(n_int) self.x_int = gle.x() self.w_int = gle.w() self.normalize()
def get_log_p_solc(): global log_p_solc from cctbx.array_family import flex from scitbx.math import chebyshev_polynome if (log_p_solc is None): coeffs = flex.double([ -14.105436736742137, -0.47015366358636385, -2.9151681976244639, -0.49308859741473005, 0.90132625209729045, 0.033529051311488103, 0.088901407582105796, 0.10749856607909694, 0.055000918494099861, -0.052424473641668454, -0.045698882840119227, 0.076048484096718036, -0.097645159906868589, 0.03904454313991608, -0.072186667173865071 ]) log_p_solc = chebyshev_polynome(15, 0, 1, coeffs) return log_p_solc
def another_example(np=41, nt=5): x = flex.double(range(np)) / (np - 1) y = 0.99 * flex.exp(-x * x * 0.5) y = -flex.log(1.0 / y - 1) w = y * y / 1.0 d = (flex.random_double(np) - 0.5) * w y_obs = y + d y = 1.0 / (1.0 + flex.exp(-y)) fit_w = chebyshev_lsq_fit.chebyshev_lsq_fit(nt, x, y_obs, w) fit_w_f = chebyshev_polynome(nt, fit_w.low_limit, fit_w.high_limit, fit_w.coefs) fit_nw = chebyshev_lsq_fit.chebyshev_lsq_fit(nt, x, y_obs) fit_nw_f = chebyshev_polynome(nt, fit_nw.low_limit, fit_nw.high_limit, fit_nw.coefs) print print "Coefficients from weighted lsq" print list(fit_w.coefs) print "Coefficients from non-weighted lsq" print list(fit_nw.coefs) assert flex.max(flex.abs(fit_nw.coefs - fit_w.coefs)) > 0
def get_log_p_solc () : global log_p_solc from cctbx.array_family import flex from scitbx.math import chebyshev_polynome if (log_p_solc is None) : coeffs = flex.double([ -14.105436736742137, -0.47015366358636385, -2.9151681976244639, -0.49308859741473005, 0.90132625209729045, 0.033529051311488103, 0.088901407582105796, 0.10749856607909694, 0.055000918494099861, -0.052424473641668454, -0.045698882840119227, 0.076048484096718036, -0.097645159906868589, 0.03904454313991608, -0.072186667173865071]) log_p_solc = chebyshev_polynome( 15, 0, 1, coeffs) return log_p_solc
def chebyshev_fit(self, x_obs, y_obs, w_obs, n_terms=None): from scitbx.math import chebyshev_polynome from scitbx.math import chebyshev_lsq_fit if n_terms is None: # determining the number of terms takes much, much longer than the fit n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=5, max_terms=20, n_goes=20, n_free=20) self.logger.info("Fitting with %i terms" %n_terms) fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs) self.logger.info("Least Squares residual: %7.6f" %(fit.f)) fit_funct = chebyshev_polynome( n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x_obs) if 0: # debugging plots from matplotlib import pyplot pyplot.clf() pyplot.plot(x_obs, y_obs) pyplot.plot(x_obs, y_fitted) pyplot.draw() pyplot.show() return y_fitted
def __init__(self, d_star_sq): ## Coefficient for gamma as a function of resolution ## described by a chebyshev polynome. coefs_mean = \ [-0.30244055359995714, 0.14551540259035137, 0.06404885418364728, 0.14126884888694674, -0.076010848339056358, 0.10445808072140797, -0.13817185173869803, 0.03162832786042917, 0.041599262771740309, -0.088562816354662108, 0.063708058411421117, -0.044796037515868393, 0.0088130627883259444, 0.02692514601906355, -0.050931900034622016, 0.019443590649642444, -0.0011195556039252301, -0.01644343506476071, 0.0065957064914017524, -0.018596655500261718, 0.0096270346410321905, 0.016307576063048754, -0.02680646640174009, 0.020734177937708331, 0.0028123353629064345, 0.0045005299107411609, 0.0076229925628943053, -0.008362403313550976, 0.0034163962268388306, 0.001904748909797396, -0.013325099196913409, 0.0048138529463141863, 0.0037576434237086738, -0.011440938719878148, 0.0070463203562045043, -0.014417892444775739, 0.00051623208479814814,-0.007030834594537072, -0.010592510032603445, 0.0099794223029419579, -0.0042803299088959388, 0.0018056147902035455, 9.1732385471614747e-05, 0.0048087303990040917, 0.0033924291685209331] coefs_mean = flex.double(coefs_mean) ## Coefficients descriubing the standard deviation of the above term coefs_sigma =\ [ 0.13066942262051989, 0.02540993472514427, 0.022640055258519923, 0.010682155584811278, 0.0055933901688389942, 0.010202224633747257, -0.0068126652213876008, 0.00074050873381034524, 0.0043056775404382332, -0.0068162235999210587, 0.0043883564931143154, -0.0046223069963272981, -0.0021799388224634842, 0.0018700994720378869, -0.0051883494911385414, -0.00036639670195559728, -0.0018731351222522098, -0.005641953724585742, -0.0021296034270177015, -0.0037654091933662288, -0.0031915331246228089, 0.0017569392295630887, -0.0023581953932665491, 0.0043374380859762538, 0.003490459547329672, 0.0030620317182512053, 0.0037626939824912907, -0.0014184248052271247, -0.0032475452005936508, -0.0053177954201788511, -0.0085157840734136816, -0.0057322608003856712, -0.0051182987317167803, -0.0052003177422633084, -0.001085721076048506, -0.00072459199543249329, 0.0010209328663554243, 0.00076695099249463397, 0.00034115347063572426, 0.0021264541997130233, -0.00031955842674212867,-0.00148958769833968, 0.0003181991857060145, -0.00069586514533741132, -0.00046211335387235546] coefs_sigma = flex.double(coefs_sigma) low = 0.008 high = 0.69 gamma_cheb = chebyshev_polynome(45, low, high, coefs_mean) gamma_sigma_cheb = chebyshev_polynome(45, low, high, coefs_sigma) ## Make sure that the d_star_sq array ## does not have any elements that fall outside ## the allowed range (determined by the range ## on which the chebyshev polynome was computed; ## i.e: self.low<=d_star_sq<=self.high d_star_sq = d_star_sq.deep_copy() lower_then_low_limit = d_star_sq <= low d_star_sq = d_star_sq.set_selected(lower_then_low_limit, low) higher_then_high_limit = d_star_sq >= high d_star_sq = d_star_sq.set_selected(higher_then_high_limit, high) ## If everything is okai, this assertion should be ## passed withgout any problem assert (flex.min(d_star_sq) >= low) assert (flex.max(d_star_sq) <= high) self.gamma = gamma_cheb.f(d_star_sq) self.sigma_gamma = gamma_sigma_cheb.f(d_star_sq)
def example(): x_obs = (flex.double(range(100)) + 1.0) / 101.0 y_ideal = flex.sin(x_obs * 6.0 * 3.1415) + flex.exp(x_obs) y_obs = y_ideal + (flex.random_double(size=x_obs.size()) - 0.5) * 0.5 w_obs = flex.double(x_obs.size(), 1) print "Trying to determine the best number of terms " print " via cross validation techniques" print n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=5, max_terms=20, n_goes=20, n_free=20) print "Fitting with", n_terms, "terms" print fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs) print "Least Squares residual: %7.6f" % (fit.f) print " R2-value : %7.6f" % (fit.f / flex.sum(y_obs * y_obs)) print fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x_obs) abs_deviation = flex.max(flex.abs((y_ideal - y_fitted))) print "Maximum deviation between fitted and error free data:" print " %4.3f" % (abs_deviation) abs_deviation = flex.mean(flex.abs((y_ideal - y_fitted))) print "Mean deviation between fitted and error free data:" print " %4.3f" % (abs_deviation) print abs_deviation = flex.max(flex.abs((y_obs - y_fitted))) print "Maximum deviation between fitted and observed data:" print " %4.3f" % (abs_deviation) abs_deviation = flex.mean(flex.abs((y_obs - y_fitted))) print "Mean deviation between fitted and observed data:" print " %4.3f" % (abs_deviation) print print "Showing 10 points" print " x y_obs y_ideal y_fit" for ii in range(10): print "%6.3f %6.3f %6.3f %6.3f" \ %(x_obs[ii*9], y_obs[ii*9], y_ideal[ii*9], y_fitted[ii*9]) try: from iotbx import data_plots except ImportError: pass else: print "Preparing output for loggraph in a file called" print " chebyshev.loggraph" chebyshev_plot = data_plots.plot_data(plot_title='Chebyshev fitting', x_label='x values', y_label='y values', x_data=x_obs, y_data=y_obs, y_legend='Observed y values', comments='Chebyshev fit') chebyshev_plot.add_data(y_data=y_ideal, y_legend='Error free y values') chebyshev_plot.add_data(y_data=y_fitted, y_legend='Fitted chebyshev approximation') output_logfile = open('chebyshev.loggraph', 'w') f = StringIO() data_plots.plot_data_loggraph(chebyshev_plot, f) output_logfile.write(f.getvalue())
def example(): x_obs = (flex.double(range(100))+1.0)/101.0 y_ideal = flex.sin(x_obs*6.0*3.1415) + flex.exp(x_obs) y_obs = y_ideal + (flex.random_double(size=x_obs.size())-0.5)*0.5 w_obs = flex.double(x_obs.size(),1) print "Trying to determine the best number of terms " print " via cross validation techniques" print n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs,y_obs,w_obs, min_terms=5 ,max_terms=20, n_goes=20,n_free=20) print "Fitting with", n_terms, "terms" print fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms,x_obs,y_obs) print "Least Squares residual: %7.6f" %(fit.f) print " R2-value : %7.6f" %(fit.f/flex.sum(y_obs*y_obs)) print fit_funct = chebyshev_polynome( n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x_obs) abs_deviation = flex.max( flex.abs( (y_ideal- y_fitted) ) ) print "Maximum deviation between fitted and error free data:" print " %4.3f" %(abs_deviation) abs_deviation = flex.mean( flex.abs( (y_ideal- y_fitted) ) ) print "Mean deviation between fitted and error free data:" print " %4.3f" %(abs_deviation) print abs_deviation = flex.max( flex.abs( (y_obs- y_fitted) ) ) print "Maximum deviation between fitted and observed data:" print " %4.3f" %(abs_deviation) abs_deviation = flex.mean( flex.abs( (y_obs- y_fitted) ) ) print "Mean deviation between fitted and observed data:" print " %4.3f" %(abs_deviation) print print "Showing 10 points" print " x y_obs y_ideal y_fit" for ii in range(10): print "%6.3f %6.3f %6.3f %6.3f" \ %(x_obs[ii*9], y_obs[ii*9], y_ideal[ii*9], y_fitted[ii*9]) try: from iotbx import data_plots except ImportError: pass else: print "Preparing output for loggraph in a file called" print " chebyshev.loggraph" chebyshev_plot = data_plots.plot_data(plot_title='Chebyshev fitting', x_label = 'x values', y_label = 'y values', x_data = x_obs, y_data = y_obs, y_legend = 'Observed y values', comments = 'Chebyshev fit') chebyshev_plot.add_data(y_data=y_ideal, y_legend='Error free y values') chebyshev_plot.add_data(y_data=y_fitted, y_legend='Fitted chebyshev approximation') output_logfile=open('chebyshev.loggraph','w') f = StringIO() data_plots.plot_data_loggraph(chebyshev_plot,f) output_logfile.write(f.getvalue())
def estimate_signal_to_noise(x, y): raise if 1: x, y = interpolate(x, y) #x, y_tr = fourier_filter(x, y) x, y_tr = savitzky_golay_filter(x, y) noise = y - y_tr else: from scitbx.math import chebyshev_polynome from scitbx.math import chebyshev_lsq_fit x_obs, y_obs = x, y w_obs = flex.double(y_obs.size(), 1) w_obs[0] = 1e16 w_obs[-1] = 1e16 ## determining the number of terms takes much, much longer than the fit n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=2, max_terms=30, n_goes=20, n_free=20) #n_terms = 7 print "n_terms:", n_terms fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs) fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit, fit.coefs) y_fitted = fit_funct.f(x) y_tr = y_fitted n = y_tr.size() noise = y - y_tr noise_sq = flex.pow2(noise) from xfel.command_line.view_pixel_histograms import sliding_average #sigma_sq = sliding_average(noise_sq, n=31) sigma_sq = sliding_average(noise_sq, n=15) #sigma_sq = sliding_average(sigma_sq) #signal_to_noise = y/flex.sqrt(sigma_sq) import math signal_to_noise = y / math.sqrt(flex.mean(noise_sq[50:200])) #pyplot.plot(noise) #pyplot.plot(x,y) #pyplot.show() offset = 0.2 * flex.max(y) offset = 0 pyplot.plot(x, y, linewidth=2) pyplot.plot(x, offset + y_tr, linewidth=2) pyplot.show() pyplot.plot(x, noise, linewidth=2) #pyplot.plot(x, flex.sqrt(sigma_sq), linewidth=2) #ax2 = pyplot.twinx() #ax2.plot(x, y) pyplot.show() pyplot.plot(x[:375], signal_to_noise[:375]) #pyplot.xlim( #ax2 = pyplot.twinx() #ax2.plot(x, y) pyplot.show()
def __init__(self, miller_obs, miller_calc, min_d_star_sq=0.0, max_d_star_sq=2.0, n_points=2000, level=6.0): assert miller_obs.indices().all_eq(miller_calc.indices()) if (miller_obs.is_xray_amplitude_array()): miller_obs = miller_obs.f_as_f_sq() if (miller_calc.is_xray_amplitude_array()): miller_calc = miller_calc.f_as_f_sq() self.obs = miller_obs.deep_copy() self.calc = miller_calc.deep_copy() self.mind = min_d_star_sq self.maxd = max_d_star_sq self.m = n_points self.n = 2 self.level = level norma_obs = absolute_scaling.kernel_normalisation( miller_array=self.obs, auto_kernel=True, n_bins=45, n_term=17) norma_calc = absolute_scaling.kernel_normalisation( miller_array=self.calc, auto_kernel=True, n_bins=45, n_term=17) obs_d_star_sq = norma_obs.d_star_sq_array calc_d_star_sq = norma_calc.d_star_sq_array sel_calc_obs = norma_calc.bin_selection.select(norma_obs.bin_selection) sel_obs_calc = norma_obs.bin_selection.select(norma_calc.bin_selection) sel = ((obs_d_star_sq > low_lim) & (obs_d_star_sq < high_lim) & (norma_obs.mean_I_array > 0)) sel = sel.select(sel_calc_obs) self.obs_d_star_sq = obs_d_star_sq.select(sel) self.calc_d_star_sq = calc_d_star_sq.select(sel_obs_calc).select(sel) self.mean_obs = norma_obs.mean_I_array.select(sel) self.mean_calc = norma_calc.mean_I_array.select(sel_obs_calc).select( sel) self.var_obs = norma_obs.var_I_array.select(sel) self.var_calc = norma_calc.var_I_array.select(sel_obs_calc).select(sel) # make an interpolator object please self.interpol = scale_curves.curve_interpolator( self.mind, self.maxd, self.m) # do the interpolation tmp_obs_d_star_sq , self.mean_obs,self.obs_a , self.obs_b = \ self.interpol.interpolate(self.obs_d_star_sq,self.mean_obs) self.obs_d_star_sq , self.var_obs,self.obs_a , self.obs_b = \ self.interpol.interpolate(self.obs_d_star_sq, self.var_obs) tmp_calc_d_star_sq , self.mean_calc,self.calc_a, self.calc_b = \ self.interpol.interpolate(self.calc_d_star_sq,self.mean_calc) self.calc_d_star_sq, self.var_calc,self.calc_a , self.calc_b = \ self.interpol.interpolate(self.calc_d_star_sq,self.var_calc) self.mean_ratio_engine = chebyshev_polynome(mean_coefs.size(), low_lim - 1e-3, high_lim + 1e-3, mean_coefs) self.std_ratio_engine = chebyshev_polynome(std_coefs.size(), low_lim - 1e-3, high_lim + 1e-3, std_coefs) self.x = flex.double([0, 0]) self.low_lim_for_scaling = 1.0 / (4.0 * 4.0) #0.0625 selection = (self.calc_d_star_sq > self.low_lim_for_scaling) if (selection.count(True) == 0): raise Sorry( "No reflections within required resolution range after " + "filtering.") self.weight_array = selection.as_double() / (2.0 * self.var_obs) assert (not self.weight_array.all_eq(0.0)) self.mean = flex.double( [1.0 / (flex.sum(self.mean_calc) / flex.sum(self.mean_obs)), 0.0]) self.sigmas = flex.double([0.5, 0.5]) s = 1.0 / (flex.sum(self.weight_array * self.mean_calc) / flex.sum(self.weight_array * self.mean_obs)) b = 0.0 self.sart_simplex = [ flex.double([s, b]), flex.double([s + 0.1, b + 1.1]), flex.double([s - 0.1, b - 1.1]) ] self.opti = simplex.simplex_opt(2, self.sart_simplex, self) sol = self.opti.get_solution() self.scale = abs(sol[0]) self.b_value = sol[1] self.modify_weights() self.all_bad_z_scores = self.weight_array.all_eq(0.0) if (not self.all_bad_z_scores): s = 1.0 / (flex.sum(self.weight_array * self.mean_calc) / flex.sum(self.weight_array * self.mean_obs)) b = 0.0 self.sart_simplex = [ flex.double([s, b]), flex.double([s + 0.1, b + 1.1]), flex.double([s - 0.1, b - 1.1]) ] self.opti = simplex.simplex_opt(2, self.sart_simplex, self)
def __init__(self, miller_array, kernel_width=None, n_bins=23, n_term=13, d_star_sq_low=None, d_star_sq_high=None, auto_kernel=False, number_of_sorted_reflections_for_auto_kernel=50): ## Autokernel is either False, true or a specific integer if kernel_width is None: assert (auto_kernel is not False) if auto_kernel is not False: assert (kernel_width==None) assert miller_array.size()>0 ## intensity arrays please work_array = None if not miller_array.is_real_array(): raise RuntimeError("Please provide real arrays only") ## I might have to change this upper condition if miller_array.is_xray_amplitude_array(): work_array = miller_array.f_as_f_sq() if miller_array.is_xray_intensity_array(): work_array = miller_array.deep_copy() work_array = work_array.set_observation_type(miller_array) ## If type is not intensity or amplitude ## raise an execption please if not miller_array.is_xray_intensity_array(): if not miller_array.is_xray_amplitude_array(): raise RuntimeError("Observation type unknown") ## declare some shorthands I_obs = work_array.data() epsilons = work_array.epsilons().data().as_double() d_star_sq_hkl = work_array.d_spacings().data() d_star_sq_hkl = 1.0/(d_star_sq_hkl*d_star_sq_hkl) ## Set up some limits if d_star_sq_low is None: d_star_sq_low = flex.min(d_star_sq_hkl) if d_star_sq_high is None: d_star_sq_high = flex.max(d_star_sq_hkl) ## A feeble attempt to determine an appropriate kernel width ## that seems to work reasonable in practice self.kernel_width=kernel_width if auto_kernel is not False: ## get the d_star_sq_array and sort it sort_permut = flex.sort_permutation(d_star_sq_hkl) ## if auto_kernel==True: number=number_of_sorted_reflections_for_auto_kernel else: number=int(auto_kernel) if number > d_star_sq_hkl.size(): number = d_star_sq_hkl.size()-1 self.kernel_width = d_star_sq_hkl[sort_permut[number]]-d_star_sq_low assert self.kernel_width > 0 ## Making the d_star_sq_array assert (n_bins>1) ## assure that there are more then 1 bins for interpolation self.d_star_sq_array = chebyshev_lsq_fit.chebyshev_nodes( n=n_bins, low=d_star_sq_low, high=d_star_sq_high, include_limits=True) ## Now get the average intensity please ## ## This step can be reasonably time consuming self.mean_I_array = scaling.kernel_normalisation( d_star_sq_hkl = d_star_sq_hkl, I_hkl = I_obs, epsilon = epsilons, d_star_sq_array = self.d_star_sq_array, kernel_width = self.kernel_width ) self.var_I_array = scaling.kernel_normalisation( d_star_sq_hkl = d_star_sq_hkl, I_hkl = I_obs*I_obs, epsilon = epsilons*epsilons, d_star_sq_array = self.d_star_sq_array, kernel_width = self.kernel_width ) self.var_I_array = self.var_I_array - self.mean_I_array*self.mean_I_array self.weight_sum = self.var_I_array = scaling.kernel_normalisation( d_star_sq_hkl = d_star_sq_hkl, I_hkl = I_obs*0.0+1.0, epsilon = epsilons*0.0+1.0, d_star_sq_array = self.d_star_sq_array, kernel_width = self.kernel_width ) eps = 1e-16 # XXX Maybe this should be larger? self.bin_selection = (self.mean_I_array > eps) sel_pos = self.bin_selection.iselection() # FIXME rare bug: this crashes when the majority of the data are zero, # e.g. because resolution limit was set too high and F/I filled in with 0. # it would be good to catch such cases in advance by inspecting the binned # values, and raise a different error message. assert sel_pos.size() > 0 if (sel_pos.size() < self.mean_I_array.size() / 2) : raise Sorry("Analysis could not be continued because more than half "+ "of the data have values below 1e-16. This usually indicates either "+ "an inappropriately high resolution cutoff, or an error in the data "+ "file which artificially creates a higher resolution limit.") self.mean_I_array = self.mean_I_array.select(sel_pos) self.d_star_sq_array = self.d_star_sq_array.select(sel_pos) self.var_I_array = flex.log( self.var_I_array.select( sel_pos ) ) self.weight_sum = self.weight_sum.select(sel_pos) self.mean_I_array = flex.log( self.mean_I_array ) ## Fit a chebyshev polynome please normalizer_fit_lsq = chebyshev_lsq_fit.chebyshev_lsq_fit( n_term, self.d_star_sq_array, self.mean_I_array ) self.normalizer = chebyshev_polynome( n_term, d_star_sq_low, d_star_sq_high, normalizer_fit_lsq.coefs) var_lsq_fit = chebyshev_lsq_fit.chebyshev_lsq_fit( n_term, self.d_star_sq_array, self.var_I_array ) self.var_norm = chebyshev_polynome( n_term, d_star_sq_low, d_star_sq_high, var_lsq_fit.coefs) ws_fit = chebyshev_lsq_fit.chebyshev_lsq_fit( n_term, self.d_star_sq_array, self.weight_sum ) self.weight_sum = chebyshev_polynome( n_term, d_star_sq_low, d_star_sq_high, ws_fit.coefs) ## The data wil now be normalised using the ## chebyshev polynome we have just obtained self.mean_I_array = flex.exp( self.mean_I_array) self.normalizer_for_miller_array = flex.exp( self.normalizer.f(d_star_sq_hkl) ) self.var_I_array = flex.exp( self.var_I_array ) self.var_norm = flex.exp( self.var_norm.f(d_star_sq_hkl) ) self.weight_sum = flex.exp( self.weight_sum.f(d_star_sq_hkl)) self.normalised_miller = None self.normalised_miller_dev_eps = None if work_array.sigmas() is not None: self.normalised_miller = work_array.customized_copy( data = work_array.data()/self.normalizer_for_miller_array, sigmas = work_array.sigmas()/self.normalizer_for_miller_array ).set_observation_type(work_array) self.normalised_miller_dev_eps = self.normalised_miller.customized_copy( data = self.normalised_miller.data()/epsilons, sigmas = self.normalised_miller.sigmas()/epsilons)\ .set_observation_type(work_array) else: self.normalised_miller = work_array.customized_copy( data = work_array.data()/self.normalizer_for_miller_array ).set_observation_type(work_array) self.normalised_miller_dev_eps = self.normalised_miller.customized_copy( data = self.normalised_miller.data()/epsilons)\ .set_observation_type(work_array)
def __init__(self, miller_obs, miller_calc, r_free_flags, kernel_width_free_reflections=None, kernel_width_d_star_cubed=None, kernel_in_bin_centers=False, kernel_on_chebyshev_nodes=True, n_sampling_points=20, n_chebyshev_terms=10, use_sampling_sum_weights=False, make_checks_and_clean_up=True): assert [kernel_width_free_reflections, kernel_width_d_star_cubed].count(None) == 1 self.miller_obs = miller_obs self.miller_calc = abs(miller_calc) self.r_free_flags = r_free_flags self.kernel_width_free_reflections = kernel_width_free_reflections self.kernel_width_d_star_cubed = kernel_width_d_star_cubed self.n_chebyshev_terms = n_chebyshev_terms if make_checks_and_clean_up: self.miller_obs = self.miller_obs.map_to_asu() self.miller_calc = self.miller_calc.map_to_asu() self.r_free_flags = self.r_free_flags.map_to_asu() assert self.r_free_flags.indices().all_eq( self.miller_obs.indices() ) self.miller_calc = self.miller_calc.common_set( self.miller_obs ) assert self.r_free_flags.indices().all_eq( self.miller_calc.indices() ) assert self.miller_obs.is_real_array() if self.miller_obs.is_xray_intensity_array(): self.miller_obs = self.miller_obs.f_sq_as_f() assert self.miller_obs.observation_type() is None or \ self.miller_obs.is_xray_amplitude_array() if self.miller_calc.observation_type() is None: self.miller_calc = self.miller_calc.set_observation_type( self.miller_obs) # get normalized data please self.normalized_obs_f = absolute_scaling.kernel_normalisation( self.miller_obs, auto_kernel=True) self.normalized_obs =self.normalized_obs_f.normalised_miller_dev_eps.f_sq_as_f() self.normalized_calc_f = absolute_scaling.kernel_normalisation( self.miller_calc, auto_kernel=True) self.normalized_calc =self.normalized_calc_f.normalised_miller_dev_eps.f_sq_as_f() # get the 'free data' if(self.r_free_flags.data().count(True) == 0): self.r_free_flags = self.r_free_flags.array( data = ~self.r_free_flags.data()) self.free_norm_obs = self.normalized_obs.select( self.r_free_flags.data() ) self.free_norm_calc= self.normalized_calc.select( self.r_free_flags.data() ) if self.free_norm_obs.data().size() <= 0: raise RuntimeError("No free reflections.") if (self.kernel_width_d_star_cubed is None): self.kernel_width_d_star_cubed=sigmaa_estimator_kernel_width_d_star_cubed( r_free_flags=self.r_free_flags, kernel_width_free_reflections=self.kernel_width_free_reflections) self.sigma_target_functor = ext.sigmaa_estimator( e_obs = self.free_norm_obs.data(), e_calc = self.free_norm_calc.data(), centric = self.free_norm_obs.centric_flags().data(), d_star_cubed = self.free_norm_obs.d_star_cubed().data() , width=self.kernel_width_d_star_cubed) d_star_cubed_overall = self.miller_obs.d_star_cubed().data() self.min_h = flex.min( d_star_cubed_overall ) self.max_h = flex.max( d_star_cubed_overall ) self.h_array = None if (kernel_in_bin_centers): self.h_array = flex.double( range(1,n_sampling_points*2,2) )*( self.max_h-self.min_h)/(n_sampling_points*2)+self.min_h else: self.min_h *= 0.99 self.max_h *= 1.01 if kernel_on_chebyshev_nodes: self.h_array = chebyshev_lsq_fit.chebyshev_nodes( n=n_sampling_points, low=self.min_h, high=self.max_h, include_limits=True) else: self.h_array = flex.double( range(n_sampling_points) )*( self.max_h-self.min_h)/float(n_sampling_points-1.0)+self.min_h assert self.h_array.size() == n_sampling_points self.sigmaa_array = flex.double() self.sigmaa_array.reserve(self.h_array.size()) self.sum_weights = flex.double() self.sum_weights.reserve(self.h_array.size()) for h in self.h_array: stimator = sigmaa_point_estimator(self.sigma_target_functor, h) self.sigmaa_array.append( stimator.sigmaa ) self.sum_weights.append( self.sigma_target_functor.sum_weights(d_star_cubed=h)) # fit a smooth function reparam_sa = -flex.log( 1.0/self.sigmaa_array -1.0 ) if (use_sampling_sum_weights): w_obs = flex.sqrt(self.sum_weights) else: w_obs = None fit_lsq = chebyshev_lsq_fit.chebyshev_lsq_fit( n_terms=self.n_chebyshev_terms, x_obs=self.h_array, y_obs=reparam_sa, w_obs=w_obs) cheb_pol = chebyshev_polynome( self.n_chebyshev_terms, self.min_h, self.max_h, fit_lsq.coefs) def reverse_reparam(values): return 1.0/(1.0 + flex.exp(-values)) self.sigmaa_fitted = reverse_reparam(cheb_pol.f(self.h_array)) self.sigmaa_miller_array = reverse_reparam(cheb_pol.f(d_star_cubed_overall)) assert flex.min(self.sigmaa_miller_array) >= 0 assert flex.max(self.sigmaa_miller_array) <= 1 self.sigmaa_miller_array = self.miller_obs.array(data=self.sigmaa_miller_array) self.alpha = None self.beta = None self.fom_array = None
def __init__(self, d_star_sq): ## Coefficient for gamma as a function of resolution ## described by a chebyshev polynome. coefs_mean = \ [-0.24994838652402987, 0.15287426147680838, 0.068108692925184011, 0.15780196907582875, -0.07811375753346686, 0.043211175909300889, -0.043407219965134192, 0.024613271516995903, 0.0035146404613345932, -0.064118486637211411, 0.10521875419321854, -0.10153928782775833, 0.0335706778430487, -0.0066629477818811282, -0.0058221659481290031, 0.0136026246654981, -0.013385834361135244, 0.022526368996167032, -0.019843844247892727, 0.018128145323325774, -0.0091740188657759101, 0.0068283902389141915, -0.0060880807366142566, 0.0004002124110802677, -0.00065686973991185187,-0.0039358839200389316, 0.0056185833386634149, -0.0075257168326962913, -0.0015215201587884459, -0.0036383549957990221, -0.0064289154284325831, 0.0059080442658917334, -0.0089851215734611887, 0.0036488156067441039, -0.0047375008148055706, -0.00090999496111171302, 0.00096986728652170276,-0.0051006830761911011, 0.0046838536228956777, -0.0031683076118337885, 0.0037866523617167236, 0.0015810274077361975, 0.0011030841357086191, 0.0015715596895281762, -0.0041354783162507788] coefs_mean = flex.double(coefs_mean) ## Coefficients descriubing the standard deviation of the above term coefs_sigma =\ [ 0.040664777671929754, 0.015978463897495004, 0.013068746157907499, -0.00022301829884293671, -3.6158446516473002e-05,-0.0038078038613494048, -0.0016909798393289835, -0.003513220109509628, -0.00097404245360874664,-0.0037187631008071421, 0.00031757305576918596,-0.0045169213215130082, -0.00086517495110945088,-0.0028285478264746034, 0.00079199093295738952, 0.00062164093803723265, 0.0018573920701968332, 0.0012407439272070957, 0.0002221210281800356, -0.00026366883273910315, -0.0011200111831549365, -0.00083301832564164021, -0.0011493805850995207, -0.00083852924805887018, -0.00019326928638570406,-0.00039548849332659371, -0.00022423676327422079,-0.00093964924566378681, -0.00063394326307545398,-0.00033546289190621823, 0.00040784160433128666, 0.001443822356327713, 0.0013545273776501506, 0.0016735119787882374, 0.0014189539521390023, 0.0013332965706491645, 0.00087782212397582277, 0.00043943299411748545, 0.00063740734290143163, 0.0007244345027539082, 0.00099161845846209391, 0.00083124705069858489, 0.00013275127763292434,-0.00023456844928215886, -0.001094278715119489] coefs_sigma = flex.double( coefs_sigma ) low = 0.008 high = 0.69 gamma_cheb=chebyshev_polynome( 45, low, high, coefs_mean) gamma_sigma_cheb = chebyshev_polynome( 45, low, high, coefs_sigma) ## Make sure that the d_star_sq array ## does not have any elements that fall outside ## the allowed range (determined by the range ## on which the chebyshev polynome was computed; ## i.e: self.low<=d_star_sq<=self.high d_star_sq = d_star_sq.deep_copy() lower_then_low_limit = d_star_sq <= low d_star_sq = d_star_sq.set_selected( lower_then_low_limit, low) higher_then_high_limit = d_star_sq >= high d_star_sq = d_star_sq.set_selected(higher_then_high_limit, high) ## If everything is okai, this assertion should be ## passed withgout any problem assert ( flex.min(d_star_sq)>=low ) assert ( flex.max(d_star_sq)<=high ) self.gamma = gamma_cheb.f( d_star_sq ) self.sigma_gamma = gamma_sigma_cheb.f( d_star_sq )
def __init__(self, d_star_sq): ## Coefficient for gamma as a function of resolution ## described by a chebyshev polynome. coefs_mean = \ [-0.30244055359995714, 0.14551540259035137, 0.06404885418364728, 0.14126884888694674, -0.076010848339056358, 0.10445808072140797, -0.13817185173869803, 0.03162832786042917, 0.041599262771740309, -0.088562816354662108, 0.063708058411421117, -0.044796037515868393, 0.0088130627883259444, 0.02692514601906355, -0.050931900034622016, 0.019443590649642444, -0.0011195556039252301, -0.01644343506476071, 0.0065957064914017524, -0.018596655500261718, 0.0096270346410321905, 0.016307576063048754, -0.02680646640174009, 0.020734177937708331, 0.0028123353629064345, 0.0045005299107411609, 0.0076229925628943053, -0.008362403313550976, 0.0034163962268388306, 0.001904748909797396, -0.013325099196913409, 0.0048138529463141863, 0.0037576434237086738, -0.011440938719878148, 0.0070463203562045043, -0.014417892444775739, 0.00051623208479814814,-0.007030834594537072, -0.010592510032603445, 0.0099794223029419579, -0.0042803299088959388, 0.0018056147902035455, 9.1732385471614747e-05, 0.0048087303990040917, 0.0033924291685209331] coefs_mean = flex.double(coefs_mean) ## Coefficients descriubing the standard deviation of the above term coefs_sigma =\ [ 0.13066942262051989, 0.02540993472514427, 0.022640055258519923, 0.010682155584811278, 0.0055933901688389942, 0.010202224633747257, -0.0068126652213876008, 0.00074050873381034524, 0.0043056775404382332, -0.0068162235999210587, 0.0043883564931143154, -0.0046223069963272981, -0.0021799388224634842, 0.0018700994720378869, -0.0051883494911385414, -0.00036639670195559728, -0.0018731351222522098, -0.005641953724585742, -0.0021296034270177015, -0.0037654091933662288, -0.0031915331246228089, 0.0017569392295630887, -0.0023581953932665491, 0.0043374380859762538, 0.003490459547329672, 0.0030620317182512053, 0.0037626939824912907, -0.0014184248052271247, -0.0032475452005936508, -0.0053177954201788511, -0.0085157840734136816, -0.0057322608003856712, -0.0051182987317167803, -0.0052003177422633084, -0.001085721076048506, -0.00072459199543249329, 0.0010209328663554243, 0.00076695099249463397, 0.00034115347063572426, 0.0021264541997130233, -0.00031955842674212867,-0.00148958769833968, 0.0003181991857060145, -0.00069586514533741132, -0.00046211335387235546] coefs_sigma = flex.double( coefs_sigma ) low = 0.008 high = 0.69 gamma_cheb=chebyshev_polynome( 45, low, high, coefs_mean) gamma_sigma_cheb = chebyshev_polynome( 45, low, high, coefs_sigma) ## Make sure that the d_star_sq array ## does not have any elements that fall outside ## the allowed range (determined by the range ## on which the chebyshev polynome was computed; ## i.e: self.low<=d_star_sq<=self.high d_star_sq = d_star_sq.deep_copy() lower_then_low_limit = d_star_sq <= low d_star_sq = d_star_sq.set_selected(lower_then_low_limit, low) higher_then_high_limit = d_star_sq >= high d_star_sq = d_star_sq.set_selected(higher_then_high_limit, high) ## If everything is okai, this assertion should be ## passed withgout any problem assert ( flex.min(d_star_sq)>=low ) assert ( flex.max(d_star_sq)<=high ) self.gamma = gamma_cheb.f( d_star_sq ) self.sigma_gamma = gamma_sigma_cheb.f( d_star_sq )
def __init__( self, miller_obs, miller_calc, r_free_flags, kernel_width_free_reflections=None, kernel_width_d_star_cubed=None, kernel_in_bin_centers=False, kernel_on_chebyshev_nodes=True, n_sampling_points=20, n_chebyshev_terms=10, use_sampling_sum_weights=False, make_checks_and_clean_up=True, ): assert [kernel_width_free_reflections, kernel_width_d_star_cubed].count(None) == 1 self.miller_obs = miller_obs self.miller_calc = abs(miller_calc) self.r_free_flags = r_free_flags self.kernel_width_free_reflections = kernel_width_free_reflections self.kernel_width_d_star_cubed = kernel_width_d_star_cubed self.n_chebyshev_terms = n_chebyshev_terms if make_checks_and_clean_up: self.miller_obs = self.miller_obs.map_to_asu() self.miller_calc = self.miller_calc.map_to_asu() self.r_free_flags = self.r_free_flags.map_to_asu() assert self.r_free_flags.indices().all_eq(self.miller_obs.indices()) self.miller_calc = self.miller_calc.common_set(self.miller_obs) assert self.r_free_flags.indices().all_eq(self.miller_calc.indices()) assert self.miller_obs.is_real_array() if self.miller_obs.is_xray_intensity_array(): self.miller_obs = self.miller_obs.f_sq_as_f() assert self.miller_obs.observation_type() is None or self.miller_obs.is_xray_amplitude_array() if self.miller_calc.observation_type() is None: self.miller_calc = self.miller_calc.set_observation_type(self.miller_obs) # get normalized data please self.normalized_obs_f = absolute_scaling.kernel_normalisation(self.miller_obs, auto_kernel=True) self.normalized_obs = self.normalized_obs_f.normalised_miller_dev_eps.f_sq_as_f() self.normalized_calc_f = absolute_scaling.kernel_normalisation(self.miller_calc, auto_kernel=True) self.normalized_calc = self.normalized_calc_f.normalised_miller_dev_eps.f_sq_as_f() # get the 'free data' if self.r_free_flags.data().count(True) == 0: self.r_free_flags = self.r_free_flags.array(data=~self.r_free_flags.data()) self.free_norm_obs = self.normalized_obs.select(self.r_free_flags.data()) self.free_norm_calc = self.normalized_calc.select(self.r_free_flags.data()) if self.free_norm_obs.data().size() <= 0: raise RuntimeError("No free reflections.") if self.kernel_width_d_star_cubed is None: self.kernel_width_d_star_cubed = sigmaa_estimator_kernel_width_d_star_cubed( r_free_flags=self.r_free_flags, kernel_width_free_reflections=self.kernel_width_free_reflections ) self.sigma_target_functor = ext.sigmaa_estimator( e_obs=self.free_norm_obs.data(), e_calc=self.free_norm_calc.data(), centric=self.free_norm_obs.centric_flags().data(), d_star_cubed=self.free_norm_obs.d_star_cubed().data(), width=self.kernel_width_d_star_cubed, ) d_star_cubed_overall = self.miller_obs.d_star_cubed().data() self.min_h = flex.min(d_star_cubed_overall) self.max_h = flex.max(d_star_cubed_overall) self.h_array = None if kernel_in_bin_centers: self.h_array = ( flex.double(xrange(1, n_sampling_points * 2, 2)) * (self.max_h - self.min_h) / (n_sampling_points * 2) + self.min_h ) else: self.min_h *= 0.99 self.max_h *= 1.01 if kernel_on_chebyshev_nodes: self.h_array = chebyshev_lsq_fit.chebyshev_nodes( n=n_sampling_points, low=self.min_h, high=self.max_h, include_limits=True ) else: self.h_array = ( flex.double(range(n_sampling_points)) * (self.max_h - self.min_h) / float(n_sampling_points - 1.0) + self.min_h ) assert self.h_array.size() == n_sampling_points self.sigmaa_array = flex.double() self.sigmaa_array.reserve(self.h_array.size()) self.sum_weights = flex.double() self.sum_weights.reserve(self.h_array.size()) for h in self.h_array: stimator = sigmaa_point_estimator(self.sigma_target_functor, h) self.sigmaa_array.append(stimator.sigmaa) self.sum_weights.append(self.sigma_target_functor.sum_weights(d_star_cubed=h)) # fit a smooth function reparam_sa = -flex.log(1.0 / self.sigmaa_array - 1.0) if use_sampling_sum_weights: w_obs = flex.sqrt(self.sum_weights) else: w_obs = None fit_lsq = chebyshev_lsq_fit.chebyshev_lsq_fit( n_terms=self.n_chebyshev_terms, x_obs=self.h_array, y_obs=reparam_sa, w_obs=w_obs ) cheb_pol = chebyshev_polynome(self.n_chebyshev_terms, self.min_h, self.max_h, fit_lsq.coefs) def reverse_reparam(values): return 1.0 / (1.0 + flex.exp(-values)) self.sigmaa_fitted = reverse_reparam(cheb_pol.f(self.h_array)) self.sigmaa_miller_array = reverse_reparam(cheb_pol.f(d_star_cubed_overall)) assert flex.min(self.sigmaa_miller_array) >= 0 assert flex.max(self.sigmaa_miller_array) <= 1 self.sigmaa_miller_array = self.miller_obs.array(data=self.sigmaa_miller_array) self.alpha = None self.beta = None self.fom_array = None
def target(self, vector): polynome = chebyshev_polynome(self.n, -1.0, 1.0, vector) self.fit_y = polynome.f(self.x) score = flex.sum(flex.pow2(self.fit_y - self.y)) return score
def __init__(self, miller_array, kernel_width=None, n_bins=23, n_term=13, d_star_sq_low=None, d_star_sq_high=None, auto_kernel=False, number_of_sorted_reflections_for_auto_kernel=50): ## Autokernel is either False, true or a specific integer if kernel_width is None: assert (auto_kernel is not False) if auto_kernel is not False: assert (kernel_width == None) assert miller_array.size() > 0 ## intensity arrays please work_array = None if not miller_array.is_real_array(): raise RuntimeError("Please provide real arrays only") ## I might have to change this upper condition if miller_array.is_xray_amplitude_array(): work_array = miller_array.f_as_f_sq() if miller_array.is_xray_intensity_array(): work_array = miller_array.deep_copy() work_array = work_array.set_observation_type(miller_array) ## If type is not intensity or amplitude ## raise an execption please if not miller_array.is_xray_intensity_array(): if not miller_array.is_xray_amplitude_array(): raise RuntimeError("Observation type unknown") ## declare some shorthands I_obs = work_array.data() epsilons = work_array.epsilons().data().as_double() d_star_sq_hkl = work_array.d_spacings().data() d_star_sq_hkl = 1.0 / (d_star_sq_hkl * d_star_sq_hkl) ## Set up some limits if d_star_sq_low is None: d_star_sq_low = flex.min(d_star_sq_hkl) if d_star_sq_high is None: d_star_sq_high = flex.max(d_star_sq_hkl) ## A feeble attempt to determine an appropriate kernel width ## that seems to work reasonable in practice self.kernel_width = kernel_width if auto_kernel is not False: ## get the d_star_sq_array and sort it sort_permut = flex.sort_permutation(d_star_sq_hkl) ## if auto_kernel == True: number = number_of_sorted_reflections_for_auto_kernel else: number = int(auto_kernel) if number > d_star_sq_hkl.size(): number = d_star_sq_hkl.size() - 1 self.kernel_width = d_star_sq_hkl[ sort_permut[number]] - d_star_sq_low assert self.kernel_width > 0 ## Making the d_star_sq_array assert (n_bins > 1 ) ## assure that there are more then 1 bins for interpolation self.d_star_sq_array = chebyshev_lsq_fit.chebyshev_nodes( n=n_bins, low=d_star_sq_low, high=d_star_sq_high, include_limits=True) ## Now get the average intensity please ## ## This step can be reasonably time consuming self.mean_I_array = scaling.kernel_normalisation( d_star_sq_hkl=d_star_sq_hkl, I_hkl=I_obs, epsilon=epsilons, d_star_sq_array=self.d_star_sq_array, kernel_width=self.kernel_width) self.var_I_array = scaling.kernel_normalisation( d_star_sq_hkl=d_star_sq_hkl, I_hkl=I_obs * I_obs, epsilon=epsilons * epsilons, d_star_sq_array=self.d_star_sq_array, kernel_width=self.kernel_width) self.var_I_array = self.var_I_array - self.mean_I_array * self.mean_I_array self.weight_sum = self.var_I_array = scaling.kernel_normalisation( d_star_sq_hkl=d_star_sq_hkl, I_hkl=I_obs * 0.0 + 1.0, epsilon=epsilons * 0.0 + 1.0, d_star_sq_array=self.d_star_sq_array, kernel_width=self.kernel_width) eps = 1e-16 # XXX Maybe this should be larger? self.bin_selection = (self.mean_I_array > eps) sel_pos = self.bin_selection.iselection() # FIXME rare bug: this crashes when the majority of the data are zero, # e.g. because resolution limit was set too high and F/I filled in with 0. # it would be good to catch such cases in advance by inspecting the binned # values, and raise a different error message. assert sel_pos.size() > 0 if (sel_pos.size() < self.mean_I_array.size() / 2): raise Sorry( "Analysis could not be continued because more than half " + "of the data have values below 1e-16. This usually indicates either " + "an inappropriately high resolution cutoff, or an error in the data " + "file which artificially creates a higher resolution limit.") self.mean_I_array = self.mean_I_array.select(sel_pos) self.d_star_sq_array = self.d_star_sq_array.select(sel_pos) self.var_I_array = flex.log(self.var_I_array.select(sel_pos)) self.weight_sum = self.weight_sum.select(sel_pos) self.mean_I_array = flex.log(self.mean_I_array) ## Fit a chebyshev polynome please normalizer_fit_lsq = chebyshev_lsq_fit.chebyshev_lsq_fit( n_term, self.d_star_sq_array, self.mean_I_array) self.normalizer = chebyshev_polynome(n_term, d_star_sq_low, d_star_sq_high, normalizer_fit_lsq.coefs) var_lsq_fit = chebyshev_lsq_fit.chebyshev_lsq_fit( n_term, self.d_star_sq_array, self.var_I_array) self.var_norm = chebyshev_polynome(n_term, d_star_sq_low, d_star_sq_high, var_lsq_fit.coefs) ws_fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_term, self.d_star_sq_array, self.weight_sum) self.weight_sum = chebyshev_polynome(n_term, d_star_sq_low, d_star_sq_high, ws_fit.coefs) ## The data wil now be normalised using the ## chebyshev polynome we have just obtained self.mean_I_array = flex.exp(self.mean_I_array) self.normalizer_for_miller_array = flex.exp( self.normalizer.f(d_star_sq_hkl)) self.var_I_array = flex.exp(self.var_I_array) self.var_norm = flex.exp(self.var_norm.f(d_star_sq_hkl)) self.weight_sum = flex.exp(self.weight_sum.f(d_star_sq_hkl)) self.normalised_miller = None self.normalised_miller_dev_eps = None if work_array.sigmas() is not None: self.normalised_miller = work_array.customized_copy( data=work_array.data() / self.normalizer_for_miller_array, sigmas=work_array.sigmas() / self.normalizer_for_miller_array).set_observation_type( work_array) self.normalised_miller_dev_eps = self.normalised_miller.customized_copy( data = self.normalised_miller.data()/epsilons, sigmas = self.normalised_miller.sigmas()/epsilons)\ .set_observation_type(work_array) else: self.normalised_miller = work_array.customized_copy( data=work_array.data() / self.normalizer_for_miller_array).set_observation_type( work_array) self.normalised_miller_dev_eps = self.normalised_miller.customized_copy( data = self.normalised_miller.data()/epsilons)\ .set_observation_type(work_array)
def __init__(self, d_star_sq): ## Coefficient for gamma as a function of resolution ## described by a chebyshev polynome. coefs_mean = \ [-0.24994838652402987, 0.15287426147680838, 0.068108692925184011, 0.15780196907582875, -0.07811375753346686, 0.043211175909300889, -0.043407219965134192, 0.024613271516995903, 0.0035146404613345932, -0.064118486637211411, 0.10521875419321854, -0.10153928782775833, 0.0335706778430487, -0.0066629477818811282, -0.0058221659481290031, 0.0136026246654981, -0.013385834361135244, 0.022526368996167032, -0.019843844247892727, 0.018128145323325774, -0.0091740188657759101, 0.0068283902389141915, -0.0060880807366142566, 0.0004002124110802677, -0.00065686973991185187,-0.0039358839200389316, 0.0056185833386634149, -0.0075257168326962913, -0.0015215201587884459, -0.0036383549957990221, -0.0064289154284325831, 0.0059080442658917334, -0.0089851215734611887, 0.0036488156067441039, -0.0047375008148055706, -0.00090999496111171302, 0.00096986728652170276,-0.0051006830761911011, 0.0046838536228956777, -0.0031683076118337885, 0.0037866523617167236, 0.0015810274077361975, 0.0011030841357086191, 0.0015715596895281762, -0.0041354783162507788] coefs_mean = flex.double(coefs_mean) ## Coefficients descriubing the standard deviation of the above term coefs_sigma =\ [ 0.040664777671929754, 0.015978463897495004, 0.013068746157907499, -0.00022301829884293671, -3.6158446516473002e-05,-0.0038078038613494048, -0.0016909798393289835, -0.003513220109509628, -0.00097404245360874664,-0.0037187631008071421, 0.00031757305576918596,-0.0045169213215130082, -0.00086517495110945088,-0.0028285478264746034, 0.00079199093295738952, 0.00062164093803723265, 0.0018573920701968332, 0.0012407439272070957, 0.0002221210281800356, -0.00026366883273910315, -0.0011200111831549365, -0.00083301832564164021, -0.0011493805850995207, -0.00083852924805887018, -0.00019326928638570406,-0.00039548849332659371, -0.00022423676327422079,-0.00093964924566378681, -0.00063394326307545398,-0.00033546289190621823, 0.00040784160433128666, 0.001443822356327713, 0.0013545273776501506, 0.0016735119787882374, 0.0014189539521390023, 0.0013332965706491645, 0.00087782212397582277, 0.00043943299411748545, 0.00063740734290143163, 0.0007244345027539082, 0.00099161845846209391, 0.00083124705069858489, 0.00013275127763292434,-0.00023456844928215886, -0.001094278715119489] coefs_sigma = flex.double(coefs_sigma) low = 0.008 high = 0.69 gamma_cheb = chebyshev_polynome(45, low, high, coefs_mean) gamma_sigma_cheb = chebyshev_polynome(45, low, high, coefs_sigma) ## Make sure that the d_star_sq array ## does not have any elements that fall outside ## the allowed range (determined by the range ## on which the chebyshev polynome was computed; ## i.e: self.low<=d_star_sq<=self.high d_star_sq = d_star_sq.deep_copy() lower_then_low_limit = d_star_sq <= low d_star_sq = d_star_sq.set_selected(lower_then_low_limit, low) higher_then_high_limit = d_star_sq >= high d_star_sq = d_star_sq.set_selected(higher_then_high_limit, high) ## If everything is okai, this assertion should be ## passed withgout any problem assert (flex.min(d_star_sq) >= low) assert (flex.max(d_star_sq) <= high) self.gamma = gamma_cheb.f(d_star_sq) self.sigma_gamma = gamma_sigma_cheb.f(d_star_sq)
def __init__(self, miller_obs, miller_calc, min_d_star_sq=0.0, max_d_star_sq=2.0, n_points=2000, level=6.0): assert miller_obs.indices().all_eq(miller_calc.indices()) if (miller_obs.is_xray_amplitude_array()) : miller_obs = miller_obs.f_as_f_sq() if (miller_calc.is_xray_amplitude_array()) : miller_calc = miller_calc.f_as_f_sq() self.obs = miller_obs.deep_copy() self.calc = miller_calc.deep_copy() self.mind = min_d_star_sq self.maxd = max_d_star_sq self.m = n_points self.n = 2 self.level = level norma_obs = absolute_scaling.kernel_normalisation( miller_array=self.obs, auto_kernel=True, n_bins=45, n_term=17) norma_calc = absolute_scaling.kernel_normalisation( miller_array=self.calc, auto_kernel=True, n_bins=45, n_term=17) obs_d_star_sq = norma_obs.d_star_sq_array calc_d_star_sq = norma_calc.d_star_sq_array sel_calc_obs = norma_calc.bin_selection.select(norma_obs.bin_selection) sel_obs_calc = norma_obs.bin_selection.select(norma_calc.bin_selection) sel = ((obs_d_star_sq > low_lim) & (obs_d_star_sq < high_lim) & (norma_obs.mean_I_array > 0)) sel = sel.select(sel_calc_obs) self.obs_d_star_sq = obs_d_star_sq.select( sel ) self.calc_d_star_sq = calc_d_star_sq.select( sel_obs_calc ).select(sel) self.mean_obs = norma_obs.mean_I_array.select(sel) self.mean_calc = norma_calc.mean_I_array.select( sel_obs_calc).select(sel) self.var_obs = norma_obs.var_I_array.select(sel) self.var_calc = norma_calc.var_I_array.select( sel_obs_calc).select(sel) # make an interpolator object please self.interpol = scale_curves.curve_interpolator( self.mind, self.maxd, self.m) # do the interpolation tmp_obs_d_star_sq , self.mean_obs,self.obs_a , self.obs_b = \ self.interpol.interpolate(self.obs_d_star_sq,self.mean_obs) self.obs_d_star_sq , self.var_obs,self.obs_a , self.obs_b = \ self.interpol.interpolate(self.obs_d_star_sq, self.var_obs) tmp_calc_d_star_sq , self.mean_calc,self.calc_a, self.calc_b = \ self.interpol.interpolate(self.calc_d_star_sq,self.mean_calc) self.calc_d_star_sq, self.var_calc,self.calc_a , self.calc_b = \ self.interpol.interpolate(self.calc_d_star_sq,self.var_calc) self.mean_ratio_engine = chebyshev_polynome( mean_coefs.size(), low_lim-1e-3, high_lim+1e-3,mean_coefs) self.std_ratio_engine = chebyshev_polynome( std_coefs.size(), low_lim-1e-3, high_lim+1e-3,std_coefs) self.x = flex.double([0,0]) self.low_lim_for_scaling = 1.0/(4.0*4.0) #0.0625 selection = (self.calc_d_star_sq > self.low_lim_for_scaling) if (selection.count(True) == 0) : raise Sorry("No reflections within required resolution range after "+ "filtering.") self.weight_array = selection.as_double() / (2.0 * self.var_obs) assert (not self.weight_array.all_eq(0.0)) self.mean = flex.double( [1.0/(flex.sum(self.mean_calc) / flex.sum(self.mean_obs)), 0.0 ] ) self.sigmas = flex.double( [0.5, 0.5] ) s = 1.0/(flex.sum(self.weight_array*self.mean_calc)/ flex.sum(self.weight_array*self.mean_obs)) b = 0.0 self.sart_simplex = [ flex.double([s,b]), flex.double([s+0.1,b+1.1]), flex.double([s-0.1,b-1.1]) ] self.opti = simplex.simplex_opt( 2, self.sart_simplex, self) sol = self.opti.get_solution() self.scale = abs(sol[0]) self.b_value = sol[1] self.modify_weights() self.all_bad_z_scores = self.weight_array.all_eq(0.0) if (not self.all_bad_z_scores) : s = 1.0/(flex.sum(self.weight_array*self.mean_calc) / flex.sum(self.weight_array*self.mean_obs)) b = 0.0 self.sart_simplex = [ flex.double([s,b]), flex.double([s+0.1,b+1.1]), flex.double([s-0.1,b-1.1]) ] self.opti = simplex.simplex_opt( 2, self.sart_simplex, self)