def runge_phenomenon(self,n=41,nt=35,print_it=False):
x_e = 2.0*(flex.double( xrange(n) )/float(n-1)-0.5)
y_e = 1/(1+x_e*x_e*25)
fit_e = chebyshev_lsq_fit.chebyshev_lsq_fit(nt,
x_e,
y_e,
)
fit_e = chebyshev_polynome(
nt, fit_e.low_limit, fit_e.high_limit, fit_e.coefs)
x_c = chebyshev_lsq_fit.chebyshev_nodes(n, -1, 1, True)
y_c = 1/(1+x_c*x_c*25)
fit_c = chebyshev_lsq_fit.chebyshev_lsq_fit(nt,
x_c,
y_c,
)
fit_c = chebyshev_polynome(
nt, fit_c.low_limit, fit_c.high_limit, fit_c.coefs)
x_plot = 2.0*(flex.double( xrange(3*n) )/float(3*n-1)-0.5)
y_plot_e = fit_e.f( x_plot )
y_plot_c = fit_c.f( x_plot )
y_id = 1/(1+x_plot*x_plot*25)
if print_it:
for x,y,yy,yyy in zip(x_plot,y_id,y_plot_e,y_plot_c):
print x,y,yy,yyy
def another_example(np=41,nt=5):
x = flex.double( range(np) )/(np-1)
y = 0.99*flex.exp(-x*x*0.5)
y = -flex.log(1.0/y-1)
w = y*y/1.0
d = (flex.random_double(np)-0.5)*w
y_obs = y+d
y = 1.0/( 1.0 + flex.exp(-y) )
fit_w = chebyshev_lsq_fit.chebyshev_lsq_fit(nt,
x,
y_obs,
w )
fit_w_f = chebyshev_polynome(
nt, fit_w.low_limit, fit_w.high_limit, fit_w.coefs)
fit_nw = chebyshev_lsq_fit.chebyshev_lsq_fit(nt,
x,
y_obs)
fit_nw_f = chebyshev_polynome(
nt, fit_nw.low_limit, fit_nw.high_limit, fit_nw.coefs)
print
print "Coefficients from weighted lsq"
print list( fit_w.coefs )
print "Coefficients from non-weighted lsq"
print list( fit_nw.coefs )
assert flex.max( flex.abs(fit_nw.coefs-fit_w.coefs) ) > 0
def runge_phenomenon(self, n=41, nt=35, print_it=False):
x_e = 2.0 * (flex.double(xrange(n)) / float(n - 1) - 0.5)
y_e = 1 / (1 + x_e * x_e * 25)
fit_e = chebyshev_lsq_fit.chebyshev_lsq_fit(
nt,
x_e,
y_e,
)
fit_e = chebyshev_polynome(nt, fit_e.low_limit, fit_e.high_limit,
fit_e.coefs)
x_c = chebyshev_lsq_fit.chebyshev_nodes(n, -1, 1, True)
y_c = 1 / (1 + x_c * x_c * 25)
fit_c = chebyshev_lsq_fit.chebyshev_lsq_fit(
nt,
x_c,
y_c,
)
fit_c = chebyshev_polynome(nt, fit_c.low_limit, fit_c.high_limit,
fit_c.coefs)
x_plot = 2.0 * (flex.double(xrange(3 * n)) / float(3 * n - 1) - 0.5)
y_plot_e = fit_e.f(x_plot)
y_plot_c = fit_c.f(x_plot)
y_id = 1 / (1 + x_plot * x_plot * 25)
if print_it:
for x, y, yy, yyy in zip(x_plot, y_id, y_plot_e, y_plot_c):
print x, y, yy, yyy
def __init__(self):
self.lims = (0, 3.999)
self.n = 25
self.cheb_coefs_mean = flex.double([
-5.7657057160113165, -1.4401697329647773, 0.27024527974695978,
0.0078985123178169463, -0.09631162819805221, 0.10735910911235932,
-0.2233791415486541, 0.19498973606201189, -0.26350484909274735,
0.27066922616706718, -0.25090257310775238, 0.22483113451869022,
-0.16726514028401993, 0.1027196386216821, -0.091673985006882924,
0.052259467525186447, -0.016970937165037378, 0.031137149395069535,
0.007431391017472774, 0.014012399241218143, 0.010042763559628737,
-0.015341614910420142, 0.0068872073662354753, 0.012869250293871155,
0.008959831186295333
])
self.cheb_coefs_low = flex.double([
-9.2669907863291705, -1.9456068148305121, 0.86779111670403164,
0.14288741649825987, 0.052348627094772476, -0.075972113378021927,
-0.24684531148041183, 0.26186454415838456, -0.11901905265242005,
0.51976185959989885, -0.45937166668405666, 0.31225783754328595,
-0.16416500741911164, 0.32535788332409726, -0.17264130742234479,
0.049992516172585558, -0.11856771807344196, 0.044221277456559308,
0.18118326484913538, 0.0550906704796576, 0.047995789360044186,
-0.083644664646019343, 0.029230535847010842, -0.040766646640578184,
-0.070401435043602661
])
self.cheb_coefs_high = flex.double([
-2.4313071167283367, -1.0123006175290996, -0.15088667146206206,
-0.13439270400442058, 0.0096372912886536471, 0.063670920315285234,
-0.21889541306029214, -0.0062901907374255505, -0.17570764254872775,
0.0052730332601467964, 0.010615988987238418, 0.14617014871482245,
0.021082968158927021, -0.0046556220561053659, -0.02634693069601177,
0.026402323365458613, -0.051317703298683792,
0.00089074796092045432, 0.011927144561737445, 0.040880441622645626,
0.028415795670163915, -0.01415133080863344, -0.02323296702274262,
-5.8561934669618548e-05, 0.051520439604993466
])
self.loc = 0.10
self.scale = 1.0
self.m = chebyshev_polynome(self.n, self.lims[0] * 0.99999,
self.lims[1] / 0.99999,
self.cheb_coefs_mean)
self.low = chebyshev_polynome(self.n, self.lims[0] * 0.99999,
self.lims[1] / 0.99999,
self.cheb_coefs_low)
self.high = chebyshev_polynome(self.n, self.lims[0] * 0.99999,
self.lims[1] / 0.99999,
self.cheb_coefs_high)
def __init__(self, x, y, order):
self.x = x
self.y = y
self.n = order
coefs = flex.random_double(self.n)
self.polynome = chebyshev_polynome(self.n, -1.0, 1.0, coefs)
self.optimize()
def chebyshev_fit(self, x_obs, y_obs, w_obs, n_terms=None):
from scitbx.math import chebyshev_polynome
from scitbx.math import chebyshev_lsq_fit
if n_terms is None:
# determining the number of terms takes much, much longer than the fit
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs,
y_obs,
w_obs,
min_terms=5,
max_terms=20,
n_goes=20,
n_free=20)
self.logger.info("Fitting with %i terms" % n_terms)
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs)
self.logger.info("Least Squares residual: %7.6f" % (fit.f))
fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit,
fit.coefs)
y_fitted = fit_funct.f(x_obs)
if 0:
# debugging plots
from matplotlib import pyplot
pyplot.clf()
pyplot.plot(x_obs, y_obs)
pyplot.plot(x_obs, y_fitted)
pyplot.draw()
pyplot.show()
return y_fitted
def normalize(self):
result = self.compute_normalisation_constant()
self.coefs[0] = self.coefs[0] - 2.0 * math.log(result)
self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs)
result = self.compute_normalisation_constant()
if abs(result - 1.0) > 1e-6:
self.normalize()
def load_coefs(self, coefs=None, random_scale=1.0, thres=600):
if coefs is not None:
assert len(self.coefs) == self.n
self.coefs = coefs
else:
self.coefs = random_scale * (flex.random_double(self.n) * 2 - 1.0)
self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs)
self.normalize()
def load_coefs(self, coefs=None):
if coefs is None:
self.coefs = (flex.random_double(self.n)-0.5)*2.0
else:
assert len(coefs)==self.n
self.coefs = coefs
# no means to refresh the coefficients yet in an elegant manner
self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs)
def load_coefs(self, coefs=None):
if coefs is None:
self.coefs = (flex.random_double(self.n) - 0.5) * 2.0
else:
assert len(coefs) == self.n
self.coefs = coefs
# no means to refresh the coefficients yet in an elegant manner
self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs)
def estimate_signal_to_noise(x, y):
raise
if 1:
x, y = interpolate(x, y)
#x, y_tr = fourier_filter(x, y)
x, y_tr = savitzky_golay_filter(x, y)
noise = y - y_tr
else:
from scitbx.math import chebyshev_polynome
from scitbx.math import chebyshev_lsq_fit
x_obs, y_obs = x, y
w_obs = flex.double(y_obs.size(), 1)
w_obs[0] = 1e16
w_obs[-1] = 1e16
## determining the number of terms takes much, much longer than the fit
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs, y_obs, w_obs,
min_terms=2, max_terms=30,
n_goes=20, n_free=20)
#n_terms = 7
print "n_terms:", n_terms
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs)
fit_funct = chebyshev_polynome(
n_terms, fit.low_limit, fit.high_limit, fit.coefs)
y_fitted = fit_funct.f(x)
y_tr = y_fitted
n = y_tr.size()
noise = y - y_tr
noise_sq = flex.pow2(noise)
from xfel.command_line.view_pixel_histograms import sliding_average
#sigma_sq = sliding_average(noise_sq, n=31)
sigma_sq = sliding_average(noise_sq, n=15)
#sigma_sq = sliding_average(sigma_sq)
#signal_to_noise = y/flex.sqrt(sigma_sq)
import math
signal_to_noise = y/math.sqrt(flex.mean(noise_sq[50:200]))
#pyplot.plot(noise)
#pyplot.plot(x,y)
#pyplot.show()
offset = 0.2 * flex.max(y)
offset = 0
pyplot.plot(x, y, linewidth=2)
pyplot.plot(x, offset+y_tr, linewidth=2)
pyplot.show()
pyplot.plot(x, noise, linewidth=2)
#pyplot.plot(x, flex.sqrt(sigma_sq), linewidth=2)
#ax2 = pyplot.twinx()
#ax2.plot(x, y)
pyplot.show()
pyplot.plot(x[:375], signal_to_noise[:375])
#pyplot.xlim(
#ax2 = pyplot.twinx()
#ax2.plot(x, y)
pyplot.show()
def calibration(pr_1, pr_2, n_params):
x1 = pr_1.r / pr_1.r[-1]
x2 = pr_2.r / pr_2.r[-1]
cdf_1 = pr_1.pr2cdf()
cdf_2 = pr_2.pr2cdf()
new_cdf_2 = flex.linear_interpolation(x2, cdf_2, x1).deep_copy()
fitting = nonlinear_fit(new_cdf_2, cdf_1, n_params)
solution = fitting.solution
fn = chebyshev_polynome(n_params, -1.0, 1.0, solution)
return fn
def __init__(self, n, m=100, k=2.5, d_max=45.0):
self.n = n
self.m = m
self.k = k
self.d_max=d_max
self.x = 1.0-2.0*(flex.double(range(m+1))/m)
self.r = 0.5*(1+self.x)*self.d_max
self.r[0] = 1e-8
self.coefs = (flex.random_double(self.n)-0.5)*0.0
self.load_coefs()
self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs)
def __init__(self, n, m=100, k=2.5, d_max=45.0):
self.n = n
self.m = m
self.k = k
self.d_max = d_max
self.x = 1.0 - 2.0 * (flex.double(range(m + 1)) / m)
self.r = 0.5 * (1 + self.x) * self.d_max
self.r[0] = 1e-8
self.coefs = (flex.random_double(self.n) - 0.5) * 0.0
self.load_coefs()
self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs)
def __init__(self, prior, n, coefs=None, n_int=25):
self.prior = prior
self.coefs = coefs
self.n = n
self.base_name = "pofx"
if self.coefs is None:
self.coefs = flex.double([0] * self.n)
self.polynome = chebyshev_polynome(self.n, -1.0, +1.0, self.coefs)
gle = scitbx.math.gauss_legendre_engine(n_int)
self.x_int = gle.x()
self.w_int = gle.w()
self.normalize()
def get_log_p_solc():
global log_p_solc
from cctbx.array_family import flex
from scitbx.math import chebyshev_polynome
if (log_p_solc is None):
coeffs = flex.double([
-14.105436736742137, -0.47015366358636385, -2.9151681976244639,
-0.49308859741473005, 0.90132625209729045, 0.033529051311488103,
0.088901407582105796, 0.10749856607909694, 0.055000918494099861,
-0.052424473641668454, -0.045698882840119227, 0.076048484096718036,
-0.097645159906868589, 0.03904454313991608, -0.072186667173865071
])
log_p_solc = chebyshev_polynome(15, 0, 1, coeffs)
return log_p_solc
def another_example(np=41, nt=5):
x = flex.double(range(np)) / (np - 1)
y = 0.99 * flex.exp(-x * x * 0.5)
y = -flex.log(1.0 / y - 1)
w = y * y / 1.0
d = (flex.random_double(np) - 0.5) * w
y_obs = y + d
y = 1.0 / (1.0 + flex.exp(-y))
fit_w = chebyshev_lsq_fit.chebyshev_lsq_fit(nt, x, y_obs, w)
fit_w_f = chebyshev_polynome(nt, fit_w.low_limit, fit_w.high_limit,
fit_w.coefs)
fit_nw = chebyshev_lsq_fit.chebyshev_lsq_fit(nt, x, y_obs)
fit_nw_f = chebyshev_polynome(nt, fit_nw.low_limit, fit_nw.high_limit,
fit_nw.coefs)
print
print "Coefficients from weighted lsq"
print list(fit_w.coefs)
print "Coefficients from non-weighted lsq"
print list(fit_nw.coefs)
assert flex.max(flex.abs(fit_nw.coefs - fit_w.coefs)) > 0
def get_log_p_solc () :
global log_p_solc
from cctbx.array_family import flex
from scitbx.math import chebyshev_polynome
if (log_p_solc is None) :
coeffs = flex.double([
-14.105436736742137, -0.47015366358636385,
-2.9151681976244639, -0.49308859741473005,
0.90132625209729045, 0.033529051311488103,
0.088901407582105796, 0.10749856607909694,
0.055000918494099861, -0.052424473641668454,
-0.045698882840119227, 0.076048484096718036,
-0.097645159906868589, 0.03904454313991608,
-0.072186667173865071])
log_p_solc = chebyshev_polynome( 15, 0, 1, coeffs)
return log_p_solc
def chebyshev_fit(self, x_obs, y_obs, w_obs, n_terms=None):
from scitbx.math import chebyshev_polynome
from scitbx.math import chebyshev_lsq_fit
if n_terms is None:
# determining the number of terms takes much, much longer than the fit
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs, y_obs, w_obs,
min_terms=5, max_terms=20,
n_goes=20, n_free=20)
self.logger.info("Fitting with %i terms" %n_terms)
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs)
self.logger.info("Least Squares residual: %7.6f" %(fit.f))
fit_funct = chebyshev_polynome(
n_terms, fit.low_limit, fit.high_limit, fit.coefs)
y_fitted = fit_funct.f(x_obs)
if 0:
# debugging plots
from matplotlib import pyplot
pyplot.clf()
pyplot.plot(x_obs, y_obs)
pyplot.plot(x_obs, y_fitted)
pyplot.draw()
pyplot.show()
return y_fitted
def __init__(self, d_star_sq):
## Coefficient for gamma as a function of resolution
## described by a chebyshev polynome.
coefs_mean = \
[-0.30244055359995714, 0.14551540259035137,
0.06404885418364728, 0.14126884888694674,
-0.076010848339056358, 0.10445808072140797,
-0.13817185173869803, 0.03162832786042917,
0.041599262771740309, -0.088562816354662108,
0.063708058411421117, -0.044796037515868393,
0.0088130627883259444, 0.02692514601906355,
-0.050931900034622016, 0.019443590649642444,
-0.0011195556039252301, -0.01644343506476071,
0.0065957064914017524, -0.018596655500261718,
0.0096270346410321905, 0.016307576063048754,
-0.02680646640174009, 0.020734177937708331,
0.0028123353629064345, 0.0045005299107411609,
0.0076229925628943053, -0.008362403313550976,
0.0034163962268388306, 0.001904748909797396,
-0.013325099196913409, 0.0048138529463141863,
0.0037576434237086738, -0.011440938719878148,
0.0070463203562045043, -0.014417892444775739,
0.00051623208479814814,-0.007030834594537072,
-0.010592510032603445, 0.0099794223029419579,
-0.0042803299088959388, 0.0018056147902035455,
9.1732385471614747e-05, 0.0048087303990040917,
0.0033924291685209331]
coefs_mean = flex.double(coefs_mean)
## Coefficients descriubing the standard deviation of the above term
coefs_sigma =\
[ 0.13066942262051989, 0.02540993472514427,
0.022640055258519923, 0.010682155584811278,
0.0055933901688389942, 0.010202224633747257,
-0.0068126652213876008, 0.00074050873381034524,
0.0043056775404382332, -0.0068162235999210587,
0.0043883564931143154, -0.0046223069963272981,
-0.0021799388224634842, 0.0018700994720378869,
-0.0051883494911385414, -0.00036639670195559728,
-0.0018731351222522098, -0.005641953724585742,
-0.0021296034270177015, -0.0037654091933662288,
-0.0031915331246228089, 0.0017569392295630887,
-0.0023581953932665491, 0.0043374380859762538,
0.003490459547329672, 0.0030620317182512053,
0.0037626939824912907, -0.0014184248052271247,
-0.0032475452005936508, -0.0053177954201788511,
-0.0085157840734136816, -0.0057322608003856712,
-0.0051182987317167803, -0.0052003177422633084,
-0.001085721076048506, -0.00072459199543249329,
0.0010209328663554243, 0.00076695099249463397,
0.00034115347063572426, 0.0021264541997130233,
-0.00031955842674212867,-0.00148958769833968,
0.0003181991857060145, -0.00069586514533741132,
-0.00046211335387235546]
coefs_sigma = flex.double(coefs_sigma)
low = 0.008
high = 0.69
gamma_cheb = chebyshev_polynome(45, low, high, coefs_mean)
gamma_sigma_cheb = chebyshev_polynome(45, low, high, coefs_sigma)
## Make sure that the d_star_sq array
## does not have any elements that fall outside
## the allowed range (determined by the range
## on which the chebyshev polynome was computed;
## i.e: self.low<=d_star_sq<=self.high
d_star_sq = d_star_sq.deep_copy()
lower_then_low_limit = d_star_sq <= low
d_star_sq = d_star_sq.set_selected(lower_then_low_limit, low)
higher_then_high_limit = d_star_sq >= high
d_star_sq = d_star_sq.set_selected(higher_then_high_limit, high)
## If everything is okai, this assertion should be
## passed withgout any problem
assert (flex.min(d_star_sq) >= low)
assert (flex.max(d_star_sq) <= high)
self.gamma = gamma_cheb.f(d_star_sq)
self.sigma_gamma = gamma_sigma_cheb.f(d_star_sq)
def example():
x_obs = (flex.double(range(100)) + 1.0) / 101.0
y_ideal = flex.sin(x_obs * 6.0 * 3.1415) + flex.exp(x_obs)
y_obs = y_ideal + (flex.random_double(size=x_obs.size()) - 0.5) * 0.5
w_obs = flex.double(x_obs.size(), 1)
print "Trying to determine the best number of terms "
print " via cross validation techniques"
print
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs, y_obs, w_obs, min_terms=5, max_terms=20, n_goes=20, n_free=20)
print "Fitting with", n_terms, "terms"
print
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs)
print "Least Squares residual: %7.6f" % (fit.f)
print " R2-value : %7.6f" % (fit.f / flex.sum(y_obs * y_obs))
print
fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit,
fit.coefs)
y_fitted = fit_funct.f(x_obs)
abs_deviation = flex.max(flex.abs((y_ideal - y_fitted)))
print "Maximum deviation between fitted and error free data:"
print " %4.3f" % (abs_deviation)
abs_deviation = flex.mean(flex.abs((y_ideal - y_fitted)))
print "Mean deviation between fitted and error free data:"
print " %4.3f" % (abs_deviation)
print
abs_deviation = flex.max(flex.abs((y_obs - y_fitted)))
print "Maximum deviation between fitted and observed data:"
print " %4.3f" % (abs_deviation)
abs_deviation = flex.mean(flex.abs((y_obs - y_fitted)))
print "Mean deviation between fitted and observed data:"
print " %4.3f" % (abs_deviation)
print
print "Showing 10 points"
print " x y_obs y_ideal y_fit"
for ii in range(10):
print "%6.3f %6.3f %6.3f %6.3f" \
%(x_obs[ii*9], y_obs[ii*9], y_ideal[ii*9], y_fitted[ii*9])
try:
from iotbx import data_plots
except ImportError:
pass
else:
print "Preparing output for loggraph in a file called"
print " chebyshev.loggraph"
chebyshev_plot = data_plots.plot_data(plot_title='Chebyshev fitting',
x_label='x values',
y_label='y values',
x_data=x_obs,
y_data=y_obs,
y_legend='Observed y values',
comments='Chebyshev fit')
chebyshev_plot.add_data(y_data=y_ideal, y_legend='Error free y values')
chebyshev_plot.add_data(y_data=y_fitted,
y_legend='Fitted chebyshev approximation')
output_logfile = open('chebyshev.loggraph', 'w')
f = StringIO()
data_plots.plot_data_loggraph(chebyshev_plot, f)
output_logfile.write(f.getvalue())
def example():
x_obs = (flex.double(range(100))+1.0)/101.0
y_ideal = flex.sin(x_obs*6.0*3.1415) + flex.exp(x_obs)
y_obs = y_ideal + (flex.random_double(size=x_obs.size())-0.5)*0.5
w_obs = flex.double(x_obs.size(),1)
print "Trying to determine the best number of terms "
print " via cross validation techniques"
print
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs,y_obs,w_obs,
min_terms=5 ,max_terms=20,
n_goes=20,n_free=20)
print "Fitting with", n_terms, "terms"
print
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms,x_obs,y_obs)
print "Least Squares residual: %7.6f" %(fit.f)
print " R2-value : %7.6f" %(fit.f/flex.sum(y_obs*y_obs))
print
fit_funct = chebyshev_polynome(
n_terms, fit.low_limit, fit.high_limit, fit.coefs)
y_fitted = fit_funct.f(x_obs)
abs_deviation = flex.max(
flex.abs( (y_ideal- y_fitted) ) )
print "Maximum deviation between fitted and error free data:"
print " %4.3f" %(abs_deviation)
abs_deviation = flex.mean(
flex.abs( (y_ideal- y_fitted) ) )
print "Mean deviation between fitted and error free data:"
print " %4.3f" %(abs_deviation)
print
abs_deviation = flex.max(
flex.abs( (y_obs- y_fitted) ) )
print "Maximum deviation between fitted and observed data:"
print " %4.3f" %(abs_deviation)
abs_deviation = flex.mean(
flex.abs( (y_obs- y_fitted) ) )
print "Mean deviation between fitted and observed data:"
print " %4.3f" %(abs_deviation)
print
print "Showing 10 points"
print " x y_obs y_ideal y_fit"
for ii in range(10):
print "%6.3f %6.3f %6.3f %6.3f" \
%(x_obs[ii*9], y_obs[ii*9], y_ideal[ii*9], y_fitted[ii*9])
try:
from iotbx import data_plots
except ImportError:
pass
else:
print "Preparing output for loggraph in a file called"
print " chebyshev.loggraph"
chebyshev_plot = data_plots.plot_data(plot_title='Chebyshev fitting',
x_label = 'x values',
y_label = 'y values',
x_data = x_obs,
y_data = y_obs,
y_legend = 'Observed y values',
comments = 'Chebyshev fit')
chebyshev_plot.add_data(y_data=y_ideal,
y_legend='Error free y values')
chebyshev_plot.add_data(y_data=y_fitted,
y_legend='Fitted chebyshev approximation')
output_logfile=open('chebyshev.loggraph','w')
f = StringIO()
data_plots.plot_data_loggraph(chebyshev_plot,f)
output_logfile.write(f.getvalue())
def estimate_signal_to_noise(x, y):
raise
if 1:
x, y = interpolate(x, y)
#x, y_tr = fourier_filter(x, y)
x, y_tr = savitzky_golay_filter(x, y)
noise = y - y_tr
else:
from scitbx.math import chebyshev_polynome
from scitbx.math import chebyshev_lsq_fit
x_obs, y_obs = x, y
w_obs = flex.double(y_obs.size(), 1)
w_obs[0] = 1e16
w_obs[-1] = 1e16
## determining the number of terms takes much, much longer than the fit
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs,
y_obs,
w_obs,
min_terms=2,
max_terms=30,
n_goes=20,
n_free=20)
#n_terms = 7
print "n_terms:", n_terms
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms, x_obs, y_obs, w_obs)
fit_funct = chebyshev_polynome(n_terms, fit.low_limit, fit.high_limit,
fit.coefs)
y_fitted = fit_funct.f(x)
y_tr = y_fitted
n = y_tr.size()
noise = y - y_tr
noise_sq = flex.pow2(noise)
from xfel.command_line.view_pixel_histograms import sliding_average
#sigma_sq = sliding_average(noise_sq, n=31)
sigma_sq = sliding_average(noise_sq, n=15)
#sigma_sq = sliding_average(sigma_sq)
#signal_to_noise = y/flex.sqrt(sigma_sq)
import math
signal_to_noise = y / math.sqrt(flex.mean(noise_sq[50:200]))
#pyplot.plot(noise)
#pyplot.plot(x,y)
#pyplot.show()
offset = 0.2 * flex.max(y)
offset = 0
pyplot.plot(x, y, linewidth=2)
pyplot.plot(x, offset + y_tr, linewidth=2)
pyplot.show()
pyplot.plot(x, noise, linewidth=2)
#pyplot.plot(x, flex.sqrt(sigma_sq), linewidth=2)
#ax2 = pyplot.twinx()
#ax2.plot(x, y)
pyplot.show()
pyplot.plot(x[:375], signal_to_noise[:375])
#pyplot.xlim(
#ax2 = pyplot.twinx()
#ax2.plot(x, y)
pyplot.show()
def __init__(self,
miller_obs,
miller_calc,
min_d_star_sq=0.0,
max_d_star_sq=2.0,
n_points=2000,
level=6.0):
assert miller_obs.indices().all_eq(miller_calc.indices())
if (miller_obs.is_xray_amplitude_array()):
miller_obs = miller_obs.f_as_f_sq()
if (miller_calc.is_xray_amplitude_array()):
miller_calc = miller_calc.f_as_f_sq()
self.obs = miller_obs.deep_copy()
self.calc = miller_calc.deep_copy()
self.mind = min_d_star_sq
self.maxd = max_d_star_sq
self.m = n_points
self.n = 2
self.level = level
norma_obs = absolute_scaling.kernel_normalisation(
miller_array=self.obs, auto_kernel=True, n_bins=45, n_term=17)
norma_calc = absolute_scaling.kernel_normalisation(
miller_array=self.calc, auto_kernel=True, n_bins=45, n_term=17)
obs_d_star_sq = norma_obs.d_star_sq_array
calc_d_star_sq = norma_calc.d_star_sq_array
sel_calc_obs = norma_calc.bin_selection.select(norma_obs.bin_selection)
sel_obs_calc = norma_obs.bin_selection.select(norma_calc.bin_selection)
sel = ((obs_d_star_sq > low_lim) & (obs_d_star_sq < high_lim) &
(norma_obs.mean_I_array > 0))
sel = sel.select(sel_calc_obs)
self.obs_d_star_sq = obs_d_star_sq.select(sel)
self.calc_d_star_sq = calc_d_star_sq.select(sel_obs_calc).select(sel)
self.mean_obs = norma_obs.mean_I_array.select(sel)
self.mean_calc = norma_calc.mean_I_array.select(sel_obs_calc).select(
sel)
self.var_obs = norma_obs.var_I_array.select(sel)
self.var_calc = norma_calc.var_I_array.select(sel_obs_calc).select(sel)
# make an interpolator object please
self.interpol = scale_curves.curve_interpolator(
self.mind, self.maxd, self.m)
# do the interpolation
tmp_obs_d_star_sq , self.mean_obs,self.obs_a , self.obs_b = \
self.interpol.interpolate(self.obs_d_star_sq,self.mean_obs)
self.obs_d_star_sq , self.var_obs,self.obs_a , self.obs_b = \
self.interpol.interpolate(self.obs_d_star_sq, self.var_obs)
tmp_calc_d_star_sq , self.mean_calc,self.calc_a, self.calc_b = \
self.interpol.interpolate(self.calc_d_star_sq,self.mean_calc)
self.calc_d_star_sq, self.var_calc,self.calc_a , self.calc_b = \
self.interpol.interpolate(self.calc_d_star_sq,self.var_calc)
self.mean_ratio_engine = chebyshev_polynome(mean_coefs.size(),
low_lim - 1e-3,
high_lim + 1e-3,
mean_coefs)
self.std_ratio_engine = chebyshev_polynome(std_coefs.size(),
low_lim - 1e-3,
high_lim + 1e-3, std_coefs)
self.x = flex.double([0, 0])
self.low_lim_for_scaling = 1.0 / (4.0 * 4.0) #0.0625
selection = (self.calc_d_star_sq > self.low_lim_for_scaling)
if (selection.count(True) == 0):
raise Sorry(
"No reflections within required resolution range after " +
"filtering.")
self.weight_array = selection.as_double() / (2.0 * self.var_obs)
assert (not self.weight_array.all_eq(0.0))
self.mean = flex.double(
[1.0 / (flex.sum(self.mean_calc) / flex.sum(self.mean_obs)), 0.0])
self.sigmas = flex.double([0.5, 0.5])
s = 1.0 / (flex.sum(self.weight_array * self.mean_calc) /
flex.sum(self.weight_array * self.mean_obs))
b = 0.0
self.sart_simplex = [
flex.double([s, b]),
flex.double([s + 0.1, b + 1.1]),
flex.double([s - 0.1, b - 1.1])
]
self.opti = simplex.simplex_opt(2, self.sart_simplex, self)
sol = self.opti.get_solution()
self.scale = abs(sol[0])
self.b_value = sol[1]
self.modify_weights()
self.all_bad_z_scores = self.weight_array.all_eq(0.0)
if (not self.all_bad_z_scores):
s = 1.0 / (flex.sum(self.weight_array * self.mean_calc) /
flex.sum(self.weight_array * self.mean_obs))
b = 0.0
self.sart_simplex = [
flex.double([s, b]),
flex.double([s + 0.1, b + 1.1]),
flex.double([s - 0.1, b - 1.1])
]
self.opti = simplex.simplex_opt(2, self.sart_simplex, self)
def __init__(self,
miller_array,
kernel_width=None,
n_bins=23,
n_term=13,
d_star_sq_low=None,
d_star_sq_high=None,
auto_kernel=False,
number_of_sorted_reflections_for_auto_kernel=50):
## Autokernel is either False, true or a specific integer
if kernel_width is None:
assert (auto_kernel is not False)
if auto_kernel is not False:
assert (kernel_width==None)
assert miller_array.size()>0
## intensity arrays please
work_array = None
if not miller_array.is_real_array():
raise RuntimeError("Please provide real arrays only")
## I might have to change this upper condition
if miller_array.is_xray_amplitude_array():
work_array = miller_array.f_as_f_sq()
if miller_array.is_xray_intensity_array():
work_array = miller_array.deep_copy()
work_array = work_array.set_observation_type(miller_array)
## If type is not intensity or amplitude
## raise an execption please
if not miller_array.is_xray_intensity_array():
if not miller_array.is_xray_amplitude_array():
raise RuntimeError("Observation type unknown")
## declare some shorthands
I_obs = work_array.data()
epsilons = work_array.epsilons().data().as_double()
d_star_sq_hkl = work_array.d_spacings().data()
d_star_sq_hkl = 1.0/(d_star_sq_hkl*d_star_sq_hkl)
## Set up some limits
if d_star_sq_low is None:
d_star_sq_low = flex.min(d_star_sq_hkl)
if d_star_sq_high is None:
d_star_sq_high = flex.max(d_star_sq_hkl)
## A feeble attempt to determine an appropriate kernel width
## that seems to work reasonable in practice
self.kernel_width=kernel_width
if auto_kernel is not False:
## get the d_star_sq_array and sort it
sort_permut = flex.sort_permutation(d_star_sq_hkl)
##
if auto_kernel==True:
number=number_of_sorted_reflections_for_auto_kernel
else:
number=int(auto_kernel)
if number > d_star_sq_hkl.size():
number = d_star_sq_hkl.size()-1
self.kernel_width = d_star_sq_hkl[sort_permut[number]]-d_star_sq_low
assert self.kernel_width > 0
## Making the d_star_sq_array
assert (n_bins>1) ## assure that there are more then 1 bins for interpolation
self.d_star_sq_array = chebyshev_lsq_fit.chebyshev_nodes(
n=n_bins,
low=d_star_sq_low,
high=d_star_sq_high,
include_limits=True)
## Now get the average intensity please
##
## This step can be reasonably time consuming
self.mean_I_array = scaling.kernel_normalisation(
d_star_sq_hkl = d_star_sq_hkl,
I_hkl = I_obs,
epsilon = epsilons,
d_star_sq_array = self.d_star_sq_array,
kernel_width = self.kernel_width
)
self.var_I_array = scaling.kernel_normalisation(
d_star_sq_hkl = d_star_sq_hkl,
I_hkl = I_obs*I_obs,
epsilon = epsilons*epsilons,
d_star_sq_array = self.d_star_sq_array,
kernel_width = self.kernel_width
)
self.var_I_array = self.var_I_array - self.mean_I_array*self.mean_I_array
self.weight_sum = self.var_I_array = scaling.kernel_normalisation(
d_star_sq_hkl = d_star_sq_hkl,
I_hkl = I_obs*0.0+1.0,
epsilon = epsilons*0.0+1.0,
d_star_sq_array = self.d_star_sq_array,
kernel_width = self.kernel_width
)
eps = 1e-16 # XXX Maybe this should be larger?
self.bin_selection = (self.mean_I_array > eps)
sel_pos = self.bin_selection.iselection()
# FIXME rare bug: this crashes when the majority of the data are zero,
# e.g. because resolution limit was set too high and F/I filled in with 0.
# it would be good to catch such cases in advance by inspecting the binned
# values, and raise a different error message.
assert sel_pos.size() > 0
if (sel_pos.size() < self.mean_I_array.size() / 2) :
raise Sorry("Analysis could not be continued because more than half "+
"of the data have values below 1e-16. This usually indicates either "+
"an inappropriately high resolution cutoff, or an error in the data "+
"file which artificially creates a higher resolution limit.")
self.mean_I_array = self.mean_I_array.select(sel_pos)
self.d_star_sq_array = self.d_star_sq_array.select(sel_pos)
self.var_I_array = flex.log( self.var_I_array.select( sel_pos ) )
self.weight_sum = self.weight_sum.select(sel_pos)
self.mean_I_array = flex.log( self.mean_I_array )
## Fit a chebyshev polynome please
normalizer_fit_lsq = chebyshev_lsq_fit.chebyshev_lsq_fit(
n_term,
self.d_star_sq_array,
self.mean_I_array )
self.normalizer = chebyshev_polynome(
n_term,
d_star_sq_low,
d_star_sq_high,
normalizer_fit_lsq.coefs)
var_lsq_fit = chebyshev_lsq_fit.chebyshev_lsq_fit(
n_term,
self.d_star_sq_array,
self.var_I_array )
self.var_norm = chebyshev_polynome(
n_term,
d_star_sq_low,
d_star_sq_high,
var_lsq_fit.coefs)
ws_fit = chebyshev_lsq_fit.chebyshev_lsq_fit(
n_term,
self.d_star_sq_array,
self.weight_sum )
self.weight_sum = chebyshev_polynome(
n_term,
d_star_sq_low,
d_star_sq_high,
ws_fit.coefs)
## The data wil now be normalised using the
## chebyshev polynome we have just obtained
self.mean_I_array = flex.exp( self.mean_I_array)
self.normalizer_for_miller_array = flex.exp( self.normalizer.f(d_star_sq_hkl) )
self.var_I_array = flex.exp( self.var_I_array )
self.var_norm = flex.exp( self.var_norm.f(d_star_sq_hkl) )
self.weight_sum = flex.exp( self.weight_sum.f(d_star_sq_hkl))
self.normalised_miller = None
self.normalised_miller_dev_eps = None
if work_array.sigmas() is not None:
self.normalised_miller = work_array.customized_copy(
data = work_array.data()/self.normalizer_for_miller_array,
sigmas = work_array.sigmas()/self.normalizer_for_miller_array
).set_observation_type(work_array)
self.normalised_miller_dev_eps = self.normalised_miller.customized_copy(
data = self.normalised_miller.data()/epsilons,
sigmas = self.normalised_miller.sigmas()/epsilons)\
.set_observation_type(work_array)
else:
self.normalised_miller = work_array.customized_copy(
data = work_array.data()/self.normalizer_for_miller_array
).set_observation_type(work_array)
self.normalised_miller_dev_eps = self.normalised_miller.customized_copy(
data = self.normalised_miller.data()/epsilons)\
.set_observation_type(work_array)
def __init__(self,
miller_obs,
miller_calc,
r_free_flags,
kernel_width_free_reflections=None,
kernel_width_d_star_cubed=None,
kernel_in_bin_centers=False,
kernel_on_chebyshev_nodes=True,
n_sampling_points=20,
n_chebyshev_terms=10,
use_sampling_sum_weights=False,
make_checks_and_clean_up=True):
assert [kernel_width_free_reflections, kernel_width_d_star_cubed].count(None) == 1
self.miller_obs = miller_obs
self.miller_calc = abs(miller_calc)
self.r_free_flags = r_free_flags
self.kernel_width_free_reflections = kernel_width_free_reflections
self.kernel_width_d_star_cubed = kernel_width_d_star_cubed
self.n_chebyshev_terms = n_chebyshev_terms
if make_checks_and_clean_up:
self.miller_obs = self.miller_obs.map_to_asu()
self.miller_calc = self.miller_calc.map_to_asu()
self.r_free_flags = self.r_free_flags.map_to_asu()
assert self.r_free_flags.indices().all_eq(
self.miller_obs.indices() )
self.miller_calc = self.miller_calc.common_set(
self.miller_obs )
assert self.r_free_flags.indices().all_eq(
self.miller_calc.indices() )
assert self.miller_obs.is_real_array()
if self.miller_obs.is_xray_intensity_array():
self.miller_obs = self.miller_obs.f_sq_as_f()
assert self.miller_obs.observation_type() is None or \
self.miller_obs.is_xray_amplitude_array()
if self.miller_calc.observation_type() is None:
self.miller_calc = self.miller_calc.set_observation_type(
self.miller_obs)
# get normalized data please
self.normalized_obs_f = absolute_scaling.kernel_normalisation(
self.miller_obs, auto_kernel=True)
self.normalized_obs =self.normalized_obs_f.normalised_miller_dev_eps.f_sq_as_f()
self.normalized_calc_f = absolute_scaling.kernel_normalisation(
self.miller_calc, auto_kernel=True)
self.normalized_calc =self.normalized_calc_f.normalised_miller_dev_eps.f_sq_as_f()
# get the 'free data'
if(self.r_free_flags.data().count(True) == 0):
self.r_free_flags = self.r_free_flags.array(
data = ~self.r_free_flags.data())
self.free_norm_obs = self.normalized_obs.select( self.r_free_flags.data() )
self.free_norm_calc= self.normalized_calc.select( self.r_free_flags.data() )
if self.free_norm_obs.data().size() <= 0:
raise RuntimeError("No free reflections.")
if (self.kernel_width_d_star_cubed is None):
self.kernel_width_d_star_cubed=sigmaa_estimator_kernel_width_d_star_cubed(
r_free_flags=self.r_free_flags,
kernel_width_free_reflections=self.kernel_width_free_reflections)
self.sigma_target_functor = ext.sigmaa_estimator(
e_obs = self.free_norm_obs.data(),
e_calc = self.free_norm_calc.data(),
centric = self.free_norm_obs.centric_flags().data(),
d_star_cubed = self.free_norm_obs.d_star_cubed().data() ,
width=self.kernel_width_d_star_cubed)
d_star_cubed_overall = self.miller_obs.d_star_cubed().data()
self.min_h = flex.min( d_star_cubed_overall )
self.max_h = flex.max( d_star_cubed_overall )
self.h_array = None
if (kernel_in_bin_centers):
self.h_array = flex.double( range(1,n_sampling_points*2,2) )*(
self.max_h-self.min_h)/(n_sampling_points*2)+self.min_h
else:
self.min_h *= 0.99
self.max_h *= 1.01
if kernel_on_chebyshev_nodes:
self.h_array = chebyshev_lsq_fit.chebyshev_nodes(
n=n_sampling_points,
low=self.min_h,
high=self.max_h,
include_limits=True)
else:
self.h_array = flex.double( range(n_sampling_points) )*(
self.max_h-self.min_h)/float(n_sampling_points-1.0)+self.min_h
assert self.h_array.size() == n_sampling_points
self.sigmaa_array = flex.double()
self.sigmaa_array.reserve(self.h_array.size())
self.sum_weights = flex.double()
self.sum_weights.reserve(self.h_array.size())
for h in self.h_array:
stimator = sigmaa_point_estimator(self.sigma_target_functor, h)
self.sigmaa_array.append( stimator.sigmaa )
self.sum_weights.append(
self.sigma_target_functor.sum_weights(d_star_cubed=h))
# fit a smooth function
reparam_sa = -flex.log( 1.0/self.sigmaa_array -1.0 )
if (use_sampling_sum_weights):
w_obs = flex.sqrt(self.sum_weights)
else:
w_obs = None
fit_lsq = chebyshev_lsq_fit.chebyshev_lsq_fit(
n_terms=self.n_chebyshev_terms,
x_obs=self.h_array,
y_obs=reparam_sa,
w_obs=w_obs)
cheb_pol = chebyshev_polynome(
self.n_chebyshev_terms,
self.min_h,
self.max_h,
fit_lsq.coefs)
def reverse_reparam(values): return 1.0/(1.0 + flex.exp(-values))
self.sigmaa_fitted = reverse_reparam(cheb_pol.f(self.h_array))
self.sigmaa_miller_array = reverse_reparam(cheb_pol.f(d_star_cubed_overall))
assert flex.min(self.sigmaa_miller_array) >= 0
assert flex.max(self.sigmaa_miller_array) <= 1
self.sigmaa_miller_array = self.miller_obs.array(data=self.sigmaa_miller_array)
self.alpha = None
self.beta = None
self.fom_array = None
def __init__(self, d_star_sq):
## Coefficient for gamma as a function of resolution
## described by a chebyshev polynome.
coefs_mean = \
[-0.24994838652402987, 0.15287426147680838,
0.068108692925184011, 0.15780196907582875,
-0.07811375753346686, 0.043211175909300889,
-0.043407219965134192, 0.024613271516995903,
0.0035146404613345932, -0.064118486637211411,
0.10521875419321854, -0.10153928782775833,
0.0335706778430487, -0.0066629477818811282,
-0.0058221659481290031, 0.0136026246654981,
-0.013385834361135244, 0.022526368996167032,
-0.019843844247892727, 0.018128145323325774,
-0.0091740188657759101, 0.0068283902389141915,
-0.0060880807366142566, 0.0004002124110802677,
-0.00065686973991185187,-0.0039358839200389316,
0.0056185833386634149, -0.0075257168326962913,
-0.0015215201587884459, -0.0036383549957990221,
-0.0064289154284325831, 0.0059080442658917334,
-0.0089851215734611887, 0.0036488156067441039,
-0.0047375008148055706, -0.00090999496111171302,
0.00096986728652170276,-0.0051006830761911011,
0.0046838536228956777, -0.0031683076118337885,
0.0037866523617167236, 0.0015810274077361975,
0.0011030841357086191, 0.0015715596895281762,
-0.0041354783162507788]
coefs_mean = flex.double(coefs_mean)
## Coefficients descriubing the standard deviation of the above term
coefs_sigma =\
[ 0.040664777671929754, 0.015978463897495004,
0.013068746157907499, -0.00022301829884293671,
-3.6158446516473002e-05,-0.0038078038613494048,
-0.0016909798393289835, -0.003513220109509628,
-0.00097404245360874664,-0.0037187631008071421,
0.00031757305576918596,-0.0045169213215130082,
-0.00086517495110945088,-0.0028285478264746034,
0.00079199093295738952, 0.00062164093803723265,
0.0018573920701968332, 0.0012407439272070957,
0.0002221210281800356, -0.00026366883273910315,
-0.0011200111831549365, -0.00083301832564164021,
-0.0011493805850995207, -0.00083852924805887018,
-0.00019326928638570406,-0.00039548849332659371,
-0.00022423676327422079,-0.00093964924566378681,
-0.00063394326307545398,-0.00033546289190621823,
0.00040784160433128666, 0.001443822356327713,
0.0013545273776501506, 0.0016735119787882374,
0.0014189539521390023, 0.0013332965706491645,
0.00087782212397582277, 0.00043943299411748545,
0.00063740734290143163, 0.0007244345027539082,
0.00099161845846209391, 0.00083124705069858489,
0.00013275127763292434,-0.00023456844928215886,
-0.001094278715119489]
coefs_sigma = flex.double( coefs_sigma )
low = 0.008
high = 0.69
gamma_cheb=chebyshev_polynome(
45, low, high, coefs_mean)
gamma_sigma_cheb = chebyshev_polynome(
45, low, high, coefs_sigma)
## Make sure that the d_star_sq array
## does not have any elements that fall outside
## the allowed range (determined by the range
## on which the chebyshev polynome was computed;
## i.e: self.low<=d_star_sq<=self.high
d_star_sq = d_star_sq.deep_copy()
lower_then_low_limit = d_star_sq <= low
d_star_sq = d_star_sq.set_selected(
lower_then_low_limit, low)
higher_then_high_limit = d_star_sq >= high
d_star_sq = d_star_sq.set_selected(higher_then_high_limit, high)
## If everything is okai, this assertion should be
## passed withgout any problem
assert ( flex.min(d_star_sq)>=low )
assert ( flex.max(d_star_sq)<=high )
self.gamma = gamma_cheb.f( d_star_sq )
self.sigma_gamma = gamma_sigma_cheb.f( d_star_sq )
def __init__(self, d_star_sq):
## Coefficient for gamma as a function of resolution
## described by a chebyshev polynome.
coefs_mean = \
[-0.30244055359995714, 0.14551540259035137,
0.06404885418364728, 0.14126884888694674,
-0.076010848339056358, 0.10445808072140797,
-0.13817185173869803, 0.03162832786042917,
0.041599262771740309, -0.088562816354662108,
0.063708058411421117, -0.044796037515868393,
0.0088130627883259444, 0.02692514601906355,
-0.050931900034622016, 0.019443590649642444,
-0.0011195556039252301, -0.01644343506476071,
0.0065957064914017524, -0.018596655500261718,
0.0096270346410321905, 0.016307576063048754,
-0.02680646640174009, 0.020734177937708331,
0.0028123353629064345, 0.0045005299107411609,
0.0076229925628943053, -0.008362403313550976,
0.0034163962268388306, 0.001904748909797396,
-0.013325099196913409, 0.0048138529463141863,
0.0037576434237086738, -0.011440938719878148,
0.0070463203562045043, -0.014417892444775739,
0.00051623208479814814,-0.007030834594537072,
-0.010592510032603445, 0.0099794223029419579,
-0.0042803299088959388, 0.0018056147902035455,
9.1732385471614747e-05, 0.0048087303990040917,
0.0033924291685209331]
coefs_mean = flex.double(coefs_mean)
## Coefficients descriubing the standard deviation of the above term
coefs_sigma =\
[ 0.13066942262051989, 0.02540993472514427,
0.022640055258519923, 0.010682155584811278,
0.0055933901688389942, 0.010202224633747257,
-0.0068126652213876008, 0.00074050873381034524,
0.0043056775404382332, -0.0068162235999210587,
0.0043883564931143154, -0.0046223069963272981,
-0.0021799388224634842, 0.0018700994720378869,
-0.0051883494911385414, -0.00036639670195559728,
-0.0018731351222522098, -0.005641953724585742,
-0.0021296034270177015, -0.0037654091933662288,
-0.0031915331246228089, 0.0017569392295630887,
-0.0023581953932665491, 0.0043374380859762538,
0.003490459547329672, 0.0030620317182512053,
0.0037626939824912907, -0.0014184248052271247,
-0.0032475452005936508, -0.0053177954201788511,
-0.0085157840734136816, -0.0057322608003856712,
-0.0051182987317167803, -0.0052003177422633084,
-0.001085721076048506, -0.00072459199543249329,
0.0010209328663554243, 0.00076695099249463397,
0.00034115347063572426, 0.0021264541997130233,
-0.00031955842674212867,-0.00148958769833968,
0.0003181991857060145, -0.00069586514533741132,
-0.00046211335387235546]
coefs_sigma = flex.double( coefs_sigma )
low = 0.008
high = 0.69
gamma_cheb=chebyshev_polynome(
45, low, high, coefs_mean)
gamma_sigma_cheb = chebyshev_polynome(
45, low, high, coefs_sigma)
## Make sure that the d_star_sq array
## does not have any elements that fall outside
## the allowed range (determined by the range
## on which the chebyshev polynome was computed;
## i.e: self.low<=d_star_sq<=self.high
d_star_sq = d_star_sq.deep_copy()
lower_then_low_limit = d_star_sq <= low
d_star_sq = d_star_sq.set_selected(lower_then_low_limit, low)
higher_then_high_limit = d_star_sq >= high
d_star_sq = d_star_sq.set_selected(higher_then_high_limit, high)
## If everything is okai, this assertion should be
## passed withgout any problem
assert ( flex.min(d_star_sq)>=low )
assert ( flex.max(d_star_sq)<=high )
self.gamma = gamma_cheb.f( d_star_sq )
self.sigma_gamma = gamma_sigma_cheb.f( d_star_sq )
def __init__(
self,
miller_obs,
miller_calc,
r_free_flags,
kernel_width_free_reflections=None,
kernel_width_d_star_cubed=None,
kernel_in_bin_centers=False,
kernel_on_chebyshev_nodes=True,
n_sampling_points=20,
n_chebyshev_terms=10,
use_sampling_sum_weights=False,
make_checks_and_clean_up=True,
):
assert [kernel_width_free_reflections, kernel_width_d_star_cubed].count(None) == 1
self.miller_obs = miller_obs
self.miller_calc = abs(miller_calc)
self.r_free_flags = r_free_flags
self.kernel_width_free_reflections = kernel_width_free_reflections
self.kernel_width_d_star_cubed = kernel_width_d_star_cubed
self.n_chebyshev_terms = n_chebyshev_terms
if make_checks_and_clean_up:
self.miller_obs = self.miller_obs.map_to_asu()
self.miller_calc = self.miller_calc.map_to_asu()
self.r_free_flags = self.r_free_flags.map_to_asu()
assert self.r_free_flags.indices().all_eq(self.miller_obs.indices())
self.miller_calc = self.miller_calc.common_set(self.miller_obs)
assert self.r_free_flags.indices().all_eq(self.miller_calc.indices())
assert self.miller_obs.is_real_array()
if self.miller_obs.is_xray_intensity_array():
self.miller_obs = self.miller_obs.f_sq_as_f()
assert self.miller_obs.observation_type() is None or self.miller_obs.is_xray_amplitude_array()
if self.miller_calc.observation_type() is None:
self.miller_calc = self.miller_calc.set_observation_type(self.miller_obs)
# get normalized data please
self.normalized_obs_f = absolute_scaling.kernel_normalisation(self.miller_obs, auto_kernel=True)
self.normalized_obs = self.normalized_obs_f.normalised_miller_dev_eps.f_sq_as_f()
self.normalized_calc_f = absolute_scaling.kernel_normalisation(self.miller_calc, auto_kernel=True)
self.normalized_calc = self.normalized_calc_f.normalised_miller_dev_eps.f_sq_as_f()
# get the 'free data'
if self.r_free_flags.data().count(True) == 0:
self.r_free_flags = self.r_free_flags.array(data=~self.r_free_flags.data())
self.free_norm_obs = self.normalized_obs.select(self.r_free_flags.data())
self.free_norm_calc = self.normalized_calc.select(self.r_free_flags.data())
if self.free_norm_obs.data().size() <= 0:
raise RuntimeError("No free reflections.")
if self.kernel_width_d_star_cubed is None:
self.kernel_width_d_star_cubed = sigmaa_estimator_kernel_width_d_star_cubed(
r_free_flags=self.r_free_flags, kernel_width_free_reflections=self.kernel_width_free_reflections
)
self.sigma_target_functor = ext.sigmaa_estimator(
e_obs=self.free_norm_obs.data(),
e_calc=self.free_norm_calc.data(),
centric=self.free_norm_obs.centric_flags().data(),
d_star_cubed=self.free_norm_obs.d_star_cubed().data(),
width=self.kernel_width_d_star_cubed,
)
d_star_cubed_overall = self.miller_obs.d_star_cubed().data()
self.min_h = flex.min(d_star_cubed_overall)
self.max_h = flex.max(d_star_cubed_overall)
self.h_array = None
if kernel_in_bin_centers:
self.h_array = (
flex.double(xrange(1, n_sampling_points * 2, 2)) * (self.max_h - self.min_h) / (n_sampling_points * 2)
+ self.min_h
)
else:
self.min_h *= 0.99
self.max_h *= 1.01
if kernel_on_chebyshev_nodes:
self.h_array = chebyshev_lsq_fit.chebyshev_nodes(
n=n_sampling_points, low=self.min_h, high=self.max_h, include_limits=True
)
else:
self.h_array = (
flex.double(range(n_sampling_points)) * (self.max_h - self.min_h) / float(n_sampling_points - 1.0)
+ self.min_h
)
assert self.h_array.size() == n_sampling_points
self.sigmaa_array = flex.double()
self.sigmaa_array.reserve(self.h_array.size())
self.sum_weights = flex.double()
self.sum_weights.reserve(self.h_array.size())
for h in self.h_array:
stimator = sigmaa_point_estimator(self.sigma_target_functor, h)
self.sigmaa_array.append(stimator.sigmaa)
self.sum_weights.append(self.sigma_target_functor.sum_weights(d_star_cubed=h))
# fit a smooth function
reparam_sa = -flex.log(1.0 / self.sigmaa_array - 1.0)
if use_sampling_sum_weights:
w_obs = flex.sqrt(self.sum_weights)
else:
w_obs = None
fit_lsq = chebyshev_lsq_fit.chebyshev_lsq_fit(
n_terms=self.n_chebyshev_terms, x_obs=self.h_array, y_obs=reparam_sa, w_obs=w_obs
)
cheb_pol = chebyshev_polynome(self.n_chebyshev_terms, self.min_h, self.max_h, fit_lsq.coefs)
def reverse_reparam(values):
return 1.0 / (1.0 + flex.exp(-values))
self.sigmaa_fitted = reverse_reparam(cheb_pol.f(self.h_array))
self.sigmaa_miller_array = reverse_reparam(cheb_pol.f(d_star_cubed_overall))
assert flex.min(self.sigmaa_miller_array) >= 0
assert flex.max(self.sigmaa_miller_array) <= 1
self.sigmaa_miller_array = self.miller_obs.array(data=self.sigmaa_miller_array)
self.alpha = None
self.beta = None
self.fom_array = None
def target(self, vector):
polynome = chebyshev_polynome(self.n, -1.0, 1.0, vector)
self.fit_y = polynome.f(self.x)
score = flex.sum(flex.pow2(self.fit_y - self.y))
return score
def __init__(self,
miller_array,
kernel_width=None,
n_bins=23,
n_term=13,
d_star_sq_low=None,
d_star_sq_high=None,
auto_kernel=False,
number_of_sorted_reflections_for_auto_kernel=50):
## Autokernel is either False, true or a specific integer
if kernel_width is None:
assert (auto_kernel is not False)
if auto_kernel is not False:
assert (kernel_width == None)
assert miller_array.size() > 0
## intensity arrays please
work_array = None
if not miller_array.is_real_array():
raise RuntimeError("Please provide real arrays only")
## I might have to change this upper condition
if miller_array.is_xray_amplitude_array():
work_array = miller_array.f_as_f_sq()
if miller_array.is_xray_intensity_array():
work_array = miller_array.deep_copy()
work_array = work_array.set_observation_type(miller_array)
## If type is not intensity or amplitude
## raise an execption please
if not miller_array.is_xray_intensity_array():
if not miller_array.is_xray_amplitude_array():
raise RuntimeError("Observation type unknown")
## declare some shorthands
I_obs = work_array.data()
epsilons = work_array.epsilons().data().as_double()
d_star_sq_hkl = work_array.d_spacings().data()
d_star_sq_hkl = 1.0 / (d_star_sq_hkl * d_star_sq_hkl)
## Set up some limits
if d_star_sq_low is None:
d_star_sq_low = flex.min(d_star_sq_hkl)
if d_star_sq_high is None:
d_star_sq_high = flex.max(d_star_sq_hkl)
## A feeble attempt to determine an appropriate kernel width
## that seems to work reasonable in practice
self.kernel_width = kernel_width
if auto_kernel is not False:
## get the d_star_sq_array and sort it
sort_permut = flex.sort_permutation(d_star_sq_hkl)
##
if auto_kernel == True:
number = number_of_sorted_reflections_for_auto_kernel
else:
number = int(auto_kernel)
if number > d_star_sq_hkl.size():
number = d_star_sq_hkl.size() - 1
self.kernel_width = d_star_sq_hkl[
sort_permut[number]] - d_star_sq_low
assert self.kernel_width > 0
## Making the d_star_sq_array
assert (n_bins > 1
) ## assure that there are more then 1 bins for interpolation
self.d_star_sq_array = chebyshev_lsq_fit.chebyshev_nodes(
n=n_bins,
low=d_star_sq_low,
high=d_star_sq_high,
include_limits=True)
## Now get the average intensity please
##
## This step can be reasonably time consuming
self.mean_I_array = scaling.kernel_normalisation(
d_star_sq_hkl=d_star_sq_hkl,
I_hkl=I_obs,
epsilon=epsilons,
d_star_sq_array=self.d_star_sq_array,
kernel_width=self.kernel_width)
self.var_I_array = scaling.kernel_normalisation(
d_star_sq_hkl=d_star_sq_hkl,
I_hkl=I_obs * I_obs,
epsilon=epsilons * epsilons,
d_star_sq_array=self.d_star_sq_array,
kernel_width=self.kernel_width)
self.var_I_array = self.var_I_array - self.mean_I_array * self.mean_I_array
self.weight_sum = self.var_I_array = scaling.kernel_normalisation(
d_star_sq_hkl=d_star_sq_hkl,
I_hkl=I_obs * 0.0 + 1.0,
epsilon=epsilons * 0.0 + 1.0,
d_star_sq_array=self.d_star_sq_array,
kernel_width=self.kernel_width)
eps = 1e-16 # XXX Maybe this should be larger?
self.bin_selection = (self.mean_I_array > eps)
sel_pos = self.bin_selection.iselection()
# FIXME rare bug: this crashes when the majority of the data are zero,
# e.g. because resolution limit was set too high and F/I filled in with 0.
# it would be good to catch such cases in advance by inspecting the binned
# values, and raise a different error message.
assert sel_pos.size() > 0
if (sel_pos.size() < self.mean_I_array.size() / 2):
raise Sorry(
"Analysis could not be continued because more than half " +
"of the data have values below 1e-16. This usually indicates either "
+
"an inappropriately high resolution cutoff, or an error in the data "
+ "file which artificially creates a higher resolution limit.")
self.mean_I_array = self.mean_I_array.select(sel_pos)
self.d_star_sq_array = self.d_star_sq_array.select(sel_pos)
self.var_I_array = flex.log(self.var_I_array.select(sel_pos))
self.weight_sum = self.weight_sum.select(sel_pos)
self.mean_I_array = flex.log(self.mean_I_array)
## Fit a chebyshev polynome please
normalizer_fit_lsq = chebyshev_lsq_fit.chebyshev_lsq_fit(
n_term, self.d_star_sq_array, self.mean_I_array)
self.normalizer = chebyshev_polynome(n_term, d_star_sq_low,
d_star_sq_high,
normalizer_fit_lsq.coefs)
var_lsq_fit = chebyshev_lsq_fit.chebyshev_lsq_fit(
n_term, self.d_star_sq_array, self.var_I_array)
self.var_norm = chebyshev_polynome(n_term, d_star_sq_low,
d_star_sq_high, var_lsq_fit.coefs)
ws_fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_term,
self.d_star_sq_array,
self.weight_sum)
self.weight_sum = chebyshev_polynome(n_term, d_star_sq_low,
d_star_sq_high, ws_fit.coefs)
## The data wil now be normalised using the
## chebyshev polynome we have just obtained
self.mean_I_array = flex.exp(self.mean_I_array)
self.normalizer_for_miller_array = flex.exp(
self.normalizer.f(d_star_sq_hkl))
self.var_I_array = flex.exp(self.var_I_array)
self.var_norm = flex.exp(self.var_norm.f(d_star_sq_hkl))
self.weight_sum = flex.exp(self.weight_sum.f(d_star_sq_hkl))
self.normalised_miller = None
self.normalised_miller_dev_eps = None
if work_array.sigmas() is not None:
self.normalised_miller = work_array.customized_copy(
data=work_array.data() / self.normalizer_for_miller_array,
sigmas=work_array.sigmas() /
self.normalizer_for_miller_array).set_observation_type(
work_array)
self.normalised_miller_dev_eps = self.normalised_miller.customized_copy(
data = self.normalised_miller.data()/epsilons,
sigmas = self.normalised_miller.sigmas()/epsilons)\
.set_observation_type(work_array)
else:
self.normalised_miller = work_array.customized_copy(
data=work_array.data() /
self.normalizer_for_miller_array).set_observation_type(
work_array)
self.normalised_miller_dev_eps = self.normalised_miller.customized_copy(
data = self.normalised_miller.data()/epsilons)\
.set_observation_type(work_array)
def __init__(self, d_star_sq):
## Coefficient for gamma as a function of resolution
## described by a chebyshev polynome.
coefs_mean = \
[-0.24994838652402987, 0.15287426147680838,
0.068108692925184011, 0.15780196907582875,
-0.07811375753346686, 0.043211175909300889,
-0.043407219965134192, 0.024613271516995903,
0.0035146404613345932, -0.064118486637211411,
0.10521875419321854, -0.10153928782775833,
0.0335706778430487, -0.0066629477818811282,
-0.0058221659481290031, 0.0136026246654981,
-0.013385834361135244, 0.022526368996167032,
-0.019843844247892727, 0.018128145323325774,
-0.0091740188657759101, 0.0068283902389141915,
-0.0060880807366142566, 0.0004002124110802677,
-0.00065686973991185187,-0.0039358839200389316,
0.0056185833386634149, -0.0075257168326962913,
-0.0015215201587884459, -0.0036383549957990221,
-0.0064289154284325831, 0.0059080442658917334,
-0.0089851215734611887, 0.0036488156067441039,
-0.0047375008148055706, -0.00090999496111171302,
0.00096986728652170276,-0.0051006830761911011,
0.0046838536228956777, -0.0031683076118337885,
0.0037866523617167236, 0.0015810274077361975,
0.0011030841357086191, 0.0015715596895281762,
-0.0041354783162507788]
coefs_mean = flex.double(coefs_mean)
## Coefficients descriubing the standard deviation of the above term
coefs_sigma =\
[ 0.040664777671929754, 0.015978463897495004,
0.013068746157907499, -0.00022301829884293671,
-3.6158446516473002e-05,-0.0038078038613494048,
-0.0016909798393289835, -0.003513220109509628,
-0.00097404245360874664,-0.0037187631008071421,
0.00031757305576918596,-0.0045169213215130082,
-0.00086517495110945088,-0.0028285478264746034,
0.00079199093295738952, 0.00062164093803723265,
0.0018573920701968332, 0.0012407439272070957,
0.0002221210281800356, -0.00026366883273910315,
-0.0011200111831549365, -0.00083301832564164021,
-0.0011493805850995207, -0.00083852924805887018,
-0.00019326928638570406,-0.00039548849332659371,
-0.00022423676327422079,-0.00093964924566378681,
-0.00063394326307545398,-0.00033546289190621823,
0.00040784160433128666, 0.001443822356327713,
0.0013545273776501506, 0.0016735119787882374,
0.0014189539521390023, 0.0013332965706491645,
0.00087782212397582277, 0.00043943299411748545,
0.00063740734290143163, 0.0007244345027539082,
0.00099161845846209391, 0.00083124705069858489,
0.00013275127763292434,-0.00023456844928215886,
-0.001094278715119489]
coefs_sigma = flex.double(coefs_sigma)
low = 0.008
high = 0.69
gamma_cheb = chebyshev_polynome(45, low, high, coefs_mean)
gamma_sigma_cheb = chebyshev_polynome(45, low, high, coefs_sigma)
## Make sure that the d_star_sq array
## does not have any elements that fall outside
## the allowed range (determined by the range
## on which the chebyshev polynome was computed;
## i.e: self.low<=d_star_sq<=self.high
d_star_sq = d_star_sq.deep_copy()
lower_then_low_limit = d_star_sq <= low
d_star_sq = d_star_sq.set_selected(lower_then_low_limit, low)
higher_then_high_limit = d_star_sq >= high
d_star_sq = d_star_sq.set_selected(higher_then_high_limit, high)
## If everything is okai, this assertion should be
## passed withgout any problem
assert (flex.min(d_star_sq) >= low)
assert (flex.max(d_star_sq) <= high)
self.gamma = gamma_cheb.f(d_star_sq)
self.sigma_gamma = gamma_sigma_cheb.f(d_star_sq)
def __init__(self,
miller_obs,
miller_calc,
min_d_star_sq=0.0,
max_d_star_sq=2.0,
n_points=2000,
level=6.0):
assert miller_obs.indices().all_eq(miller_calc.indices())
if (miller_obs.is_xray_amplitude_array()) :
miller_obs = miller_obs.f_as_f_sq()
if (miller_calc.is_xray_amplitude_array()) :
miller_calc = miller_calc.f_as_f_sq()
self.obs = miller_obs.deep_copy()
self.calc = miller_calc.deep_copy()
self.mind = min_d_star_sq
self.maxd = max_d_star_sq
self.m = n_points
self.n = 2
self.level = level
norma_obs = absolute_scaling.kernel_normalisation(
miller_array=self.obs,
auto_kernel=True,
n_bins=45,
n_term=17)
norma_calc = absolute_scaling.kernel_normalisation(
miller_array=self.calc,
auto_kernel=True,
n_bins=45,
n_term=17)
obs_d_star_sq = norma_obs.d_star_sq_array
calc_d_star_sq = norma_calc.d_star_sq_array
sel_calc_obs = norma_calc.bin_selection.select(norma_obs.bin_selection)
sel_obs_calc = norma_obs.bin_selection.select(norma_calc.bin_selection)
sel = ((obs_d_star_sq > low_lim) & (obs_d_star_sq < high_lim) &
(norma_obs.mean_I_array > 0))
sel = sel.select(sel_calc_obs)
self.obs_d_star_sq = obs_d_star_sq.select( sel )
self.calc_d_star_sq = calc_d_star_sq.select( sel_obs_calc ).select(sel)
self.mean_obs = norma_obs.mean_I_array.select(sel)
self.mean_calc = norma_calc.mean_I_array.select(
sel_obs_calc).select(sel)
self.var_obs = norma_obs.var_I_array.select(sel)
self.var_calc = norma_calc.var_I_array.select(
sel_obs_calc).select(sel)
# make an interpolator object please
self.interpol = scale_curves.curve_interpolator( self.mind, self.maxd,
self.m)
# do the interpolation
tmp_obs_d_star_sq , self.mean_obs,self.obs_a , self.obs_b = \
self.interpol.interpolate(self.obs_d_star_sq,self.mean_obs)
self.obs_d_star_sq , self.var_obs,self.obs_a , self.obs_b = \
self.interpol.interpolate(self.obs_d_star_sq, self.var_obs)
tmp_calc_d_star_sq , self.mean_calc,self.calc_a, self.calc_b = \
self.interpol.interpolate(self.calc_d_star_sq,self.mean_calc)
self.calc_d_star_sq, self.var_calc,self.calc_a , self.calc_b = \
self.interpol.interpolate(self.calc_d_star_sq,self.var_calc)
self.mean_ratio_engine = chebyshev_polynome( mean_coefs.size(),
low_lim-1e-3, high_lim+1e-3,mean_coefs)
self.std_ratio_engine = chebyshev_polynome( std_coefs.size(),
low_lim-1e-3, high_lim+1e-3,std_coefs)
self.x = flex.double([0,0])
self.low_lim_for_scaling = 1.0/(4.0*4.0) #0.0625
selection = (self.calc_d_star_sq > self.low_lim_for_scaling)
if (selection.count(True) == 0) :
raise Sorry("No reflections within required resolution range after "+
"filtering.")
self.weight_array = selection.as_double() / (2.0 * self.var_obs)
assert (not self.weight_array.all_eq(0.0))
self.mean = flex.double( [1.0/(flex.sum(self.mean_calc) /
flex.sum(self.mean_obs)), 0.0 ] )
self.sigmas = flex.double( [0.5, 0.5] )
s = 1.0/(flex.sum(self.weight_array*self.mean_calc)/
flex.sum(self.weight_array*self.mean_obs))
b = 0.0
self.sart_simplex = [ flex.double([s,b]), flex.double([s+0.1,b+1.1]),
flex.double([s-0.1,b-1.1]) ]
self.opti = simplex.simplex_opt( 2, self.sart_simplex, self)
sol = self.opti.get_solution()
self.scale = abs(sol[0])
self.b_value = sol[1]
self.modify_weights()
self.all_bad_z_scores = self.weight_array.all_eq(0.0)
if (not self.all_bad_z_scores) :
s = 1.0/(flex.sum(self.weight_array*self.mean_calc) /
flex.sum(self.weight_array*self.mean_obs))
b = 0.0
self.sart_simplex = [ flex.double([s,b]), flex.double([s+0.1,b+1.1]),
flex.double([s-0.1,b-1.1]) ]
self.opti = simplex.simplex_opt( 2, self.sart_simplex, self)
