def partial_derivatives(self, x_obs):
shape, location, scale = self.params
exponential_part = (1 / (math.sqrt(2 * math.pi)) *
flex.exp(-flex.pow2(x_obs - location) /
(2 * scale**2)))
normal_part = 2 / scale * exponential_part
cdf_part = 0.5 * (1 + scitbx.math.erf(shape * (x_obs - location) /
(math.sqrt(2) * scale)))
d_normal_part_d_location = 2 / scale**3 * (x_obs -
location) * exponential_part
d_normal_part_d_scale = \
2 / scale**4 * (flex.pow2(x_obs - location) - scale**2) * exponential_part
exponential_part_with_shape = (
1 / (math.sqrt(math.pi)) *
flex.exp(-shape**2 * flex.pow2(x_obs - location) / (2 * scale**2)))
d_cdf_d_shape = \
(x_obs - location) / (math.sqrt(2) * scale) * exponential_part_with_shape
d_cdf_d_location = \
-shape / (math.sqrt(2) * scale) * exponential_part_with_shape
d_cdf_d_scale = (-shape * (x_obs - location) *
exponential_part_with_shape /
(math.sqrt(2) * scale**2))
# product rule
return (d_cdf_d_shape * normal_part,
d_normal_part_d_location * cdf_part +
d_cdf_d_location * normal_part,
d_normal_part_d_scale * cdf_part + d_cdf_d_scale * normal_part)
def target(self, vector):
v = 1.0
scale = abs(vector[0])
b_value = vector[1]
if b_value > 200.0:
b_value = 200.0
if b_value < -200.0:
b_value = -200.0
i_scaled = flex.exp(
self.calc_d_star_sq * b_value) * self.mean_calc * scale
ratio = i_scaled / self.mean_obs
curve = self.curve(self.calc_d_star_sq)
result = ratio - flex.exp(curve)
if (flex.max(result) > math.sqrt(sys.float_info.max)):
raise OverflowError("Result array exceeds floating-point limit.")
result = result * result
wmax = flex.max(self.weight_array)
assert (wmax != 0)
if (wmax > 1) and (flex.max(result) > sys.float_info.max / wmax):
raise OverflowError(
"Weighted result array will exceed floating-point " +
"limit: %e" % flex.max(result))
result = result * self.weight_array
result = flex.sum(result)
return result
def __init__(self, xray_structure, k_anisotropic, k_masks, ss):
self.xray_structure = xray_structure
self.k_anisotropic = k_anisotropic
self.k_masks = k_masks
self.ss = ss
#
k_total = self.k_anisotropic
r = scitbx.math.gaussian_fit_1d_analytical(x=flex.sqrt(self.ss), y=k_total)
k,b = r.a, r.b
#
k,b,r = mmtbx.bulk_solvent.fit_k_exp_b_to_k_total(k_total, self.ss, k, b)
k_exp_overall, b_exp_overall = None,None
if(r<0.7): k_exp_overall, b_exp_overall = k,b
if(self.xray_structure is None): return None
b_adj = 0
if([k_exp_overall, b_exp_overall].count(None)==0 and k != 0):
bs1 = self.xray_structure.extract_u_iso_or_u_equiv()*adptbx.u_as_b(1.)
def split(b_trace, xray_structure):
b_min = xray_structure.min_u_cart_eigenvalue()*adptbx.u_as_b(1.)
b_res = min(0, b_min + b_trace+1.e-6)
b_adj = b_trace-b_res
xray_structure.shift_us(b_shift = b_adj)
return b_adj, b_res
b_adj,b_res=split(b_trace=b_exp_overall,xray_structure=self.xray_structure)
k_new = k_exp_overall*flex.exp(-self.ss*b_adj)
bs2 = self.xray_structure.extract_u_iso_or_u_equiv()*adptbx.u_as_b(1.)
diff = bs2-bs1
assert approx_equal(flex.min(diff), flex.max(diff))
assert approx_equal(flex.max(diff), b_adj)
self.k_anisotropic = self.k_anisotropic/k_new
self.k_masks = [m*flex.exp(-self.ss*b_adj) for m in self.k_masks]
def another_example(np=41,nt=5):
x = flex.double( range(np) )/(np-1)
y = 0.99*flex.exp(-x*x*0.5)
y = -flex.log(1.0/y-1)
w = y*y/1.0
d = (flex.random_double(np)-0.5)*w
y_obs = y+d
y = 1.0/( 1.0 + flex.exp(-y) )
fit_w = chebyshev_lsq_fit.chebyshev_lsq_fit(nt,
x,
y_obs,
w )
fit_w_f = chebyshev_polynome(
nt, fit_w.low_limit, fit_w.high_limit, fit_w.coefs)
fit_nw = chebyshev_lsq_fit.chebyshev_lsq_fit(nt,
x,
y_obs)
fit_nw_f = chebyshev_polynome(
nt, fit_nw.low_limit, fit_nw.high_limit, fit_nw.coefs)
print
print "Coefficients from weighted lsq"
print list( fit_w.coefs )
print "Coefficients from non-weighted lsq"
print list( fit_nw.coefs )
assert flex.max( flex.abs(fit_nw.coefs-fit_w.coefs) ) > 0
def apply_back_trace_of_overall_exp_scale_matrix(self, xray_structure=None):
k,b=self.overall_isotropic_kb_estimate()
k_total = self.core.k_isotropic * self.core.k_anisotropic * \
self.core.k_isotropic_exp
k,b,r = mmtbx.bulk_solvent.fit_k_exp_b_to_k_total(k_total, self.ss, k, b)
if(r<0.7): self.k_exp_overall,self.b_exp_overall = k,b
if(xray_structure is None): return None
b_adj = 0
if([self.k_exp_overall,self.b_exp_overall].count(None)==0 and k != 0):
bs1 = xray_structure.extract_u_iso_or_u_equiv()*adptbx.u_as_b(1.)
def split(b_trace, xray_structure):
b_min = xray_structure.min_u_cart_eigenvalue()*adptbx.u_as_b(1.)
b_res = min(0, b_min + b_trace+1.e-6)
b_adj = b_trace-b_res
xray_structure.shift_us(b_shift = b_adj)
return b_adj, b_res
b_adj,b_res=split(b_trace=self.b_exp_overall,xray_structure=xray_structure)
k_new = self.k_exp_overall*flex.exp(-self.ss*b_adj)
bs2 = xray_structure.extract_u_iso_or_u_equiv()*adptbx.u_as_b(1.)
diff = bs2-bs1
assert approx_equal(flex.min(diff), flex.max(diff))
assert approx_equal(flex.max(diff), b_adj)
self.core = self.core.update(
k_isotropic = self.core.k_isotropic,
k_isotropic_exp = self.core.k_isotropic_exp/k_new,
k_masks = [m*flex.exp(-self.ss*b_adj) for m in self.core.k_masks])
return group_args(
xray_structure = xray_structure,
k_isotropic = self.k_isotropic(),
k_anisotropic = self.k_anisotropic(),
k_mask = self.k_masks(),
b_adj = b_adj)
def b_factor_sharpening_by_map_kurtosis_maximization(map_coeffs, show=True,
b_sharp_best=None, b_only=False):
ss = 1./flex.pow2(map_coeffs.d_spacings().data()) / 4.
if(b_sharp_best is None):
b_sharp_best = None
kurt = -999
for b_sharp in range(-100,100,5):
k_sharp = 1./flex.exp(-ss * b_sharp)
map_coeffs_ = map_coeffs.deep_copy().customized_copy(
data = map_coeffs.data()*k_sharp)
fft_map = map_coeffs_.fft_map(resolution_factor = 0.25)
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded()
o = maptbx.more_statistics(map_data)
kurt_ = o.kurtosis()
if(kurt_ > kurt):
kurt = kurt_
b_sharp_best = b_sharp
if(show):
print "b_sharp: %6.1f skewness: %6.4f kurtosis: %6.4f"%(b_sharp,
o.skewness(), o.kurtosis())
if(show): print "Best sharpening B-factor:", b_sharp_best
k_sharp = 1./flex.exp(-ss * b_sharp_best)
if(b_only): return b_sharp_best
else:
return map_coeffs.customized_copy(data = map_coeffs.data()*k_sharp)
def functional(self):
IAcurr, IBcurr, Gcurr = self.unpack()
self.resid = self.Yobs - Gcurr * (
flex.exp(IAcurr) * self.LA * self.PA +
flex.exp(IBcurr) * self.LB * self.PB)
resid_sq = self.resid * self.resid
return .5 * flex.sum(resid_sq)
def b_factor_sharpening_by_map_kurtosis_maximization(map_coeffs,
show=True,
b_sharp_best=None,
b_only=False,
b_min=-100,
b_max=100,
b_step=5):
ss = 1. / flex.pow2(map_coeffs.d_spacings().data()) / 4.
if (b_sharp_best is None):
b_sharp_best = None
kurt = -999
for b_sharp in range(b_min, b_max, b_step):
k_sharp = 1. / flex.exp(-ss * b_sharp)
map_coeffs_ = map_coeffs.deep_copy().customized_copy(
data=map_coeffs.data() * k_sharp)
fft_map = map_coeffs_.fft_map(resolution_factor=0.25)
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded()
o = maptbx.more_statistics(map_data)
kurt_ = o.kurtosis()
if (kurt_ > kurt):
kurt = kurt_
b_sharp_best = b_sharp
if (show):
print "b_sharp: %6.1f skewness: %6.4f kurtosis: %6.4f" % (
b_sharp, o.skewness(), o.kurtosis())
if (show): print "Best sharpening B-factor:", b_sharp_best
k_sharp = 1. / flex.exp(-ss * b_sharp_best)
if (b_only): return b_sharp_best
else:
return map_coeffs.customized_copy(data=map_coeffs.data() * k_sharp)
def single_level_curve(self, gi, rgi, bi, rgcfi, pi, q=None):
gi = abs(gi)
rgi = abs(rgi)
bi = abs(bi)
rgcfi = abs(rgcfi)
if q is None:
q = self.q
result = abs(gi) * flex.exp(-q * q * rgi * rgi / 3.0) + bi * flex.exp(
-q * q * rgcfi * rgcfi / 3.0) * flex.pow(
flex.pow(sm.erf(q * rgi * self.cnst), 3.0) /
(q + self.eps), pi)
return result
def set_k_isotropic_exp(self, r_start, verbose, b_lower_limit=-100):
if (self.verbose):
print(" set_k_isotropic_exp:", file=self.log)
print(" r_start: %6.4f (r_low: %6.4f)" %
(r_start, self._r_low()))
k_iso = flex.double(self.core.k_isotropic.size(),
1) # Done at start only!
k_aniso = flex.double(self.core.k_isotropic.size(),
1) # Done at start only!
arrays = mmtbx.arrays.init(f_calc=self.core.f_calc,
f_masks=self.core.f_mask(),
k_isotropic=k_iso,
k_anisotropic=k_aniso,
k_masks=self.core.k_mask())
sel = self.selection_work.data()
#
# At least in one example this gives more accurate answer but higher R than start!
#
rf = scitbx.math.gaussian_fit_1d_analytical(
x=flex.sqrt(self.ss).select(sel),
y=self.f_obs.data().select(sel),
z=abs(arrays.f_model).data().select(sel))
if (rf.b < b_lower_limit): return r_start
k1 = rf.a * flex.exp(-self.ss * rf.b)
r1 = self.try_scale(k_isotropic_exp=k1)
#
# At least in one example this gives less accurate answer but lower R than start!
#
o = bulk_solvent.f_kb_scaled(f1=self.f_obs.data().select(sel),
f2=flex.abs(
arrays.f_model.data()).select(sel),
b_range=flex.double(range(-100, 100, 1)),
ss=self.ss.select(sel))
k2 = o.k() * flex.exp(-self.ss * o.b())
r2 = self.try_scale(k_isotropic_exp=k2)
#
if (r1 < r2):
r = r1
k = k1
else:
r = r2
k = k2
if (r < r_start):
self.core = self.core.update(k_isotropic_exp=k)
r = self.r_factor()
if (self.verbose):
print(" r1: %6.4f" % r1)
print(" r2: %6.4f" % r2)
print(" r_final: %6.4f (r_low: %6.4f)" % (r, self._r_low()))
return r
def target(self, vector):
self.counter += 1
result = 0
length = flex.sum(flex.pow(vector, 2))
if (length > self.Rmax2):
result = 100000000000000
else:
new_coord = self.pdb.NMPerturb(self.modes, vector)
self.pdb.model.updateDistArray(flex.vec3_double(new_coord))
dist_array = self.pdb.model.getDistArray()
new_pr = self.pdb.model.Histogram(dist_array, self.dMax,
self.n_slot)
if (self.weighted):
if (self.scale == 0):
self.scale = float(flex.mean(
flex.abs(self.expt - new_pr))) / float(
flex.mean(self.expt))
self.scale2 = self.scale * self.scale
result = flex.pow((self.expt - new_pr), 2.0)
result = flex.sum(
flex.exp(-self.scale2 * self.expt2 / (result + 10 - 12)) *
result)
else:
result = flex.sum(flex.pow((self.expt - new_pr), 2.0))
result = result * 100
return result
def get_rmsd_from_lddt(lddt_values, is_fractional=None):
"""
get_rmsd_from_lddt:
Purpose: AlphaFold models are supplied with values of LDDT (confidence)
in the B-value field. This routine uses the formula
from the alphafold paper to convert these values to error estimates.
NOTE: lddt_values can be fractional (0 to 1) or percentage (0 to 100)
If is_fractional is not set, assume fractional if all between 0 and 1
rmsd_est = 1.5 * flex.exp(4*(0.7-fractional_values))
NOTE: formulas taken from phaser_voyager implementation by
Claudia Millan and Massimo Sammito at
phaser_voyager/src/Voyager/MDSLibraries/pdb_structure.py
Inputs:
lddt_values: flex array of lddt values
Outputs:
flex array of error estimates (A)
"""
if is_fractional is None:
is_fractional = (lddt_values.min_max_mean().min >= 0
and lddt_values.min_max_mean().max <= 1)
if is_fractional:
fractional_values = lddt_values.deep_copy()
else:
fractional_values = lddt_values * 0.01
fractional_values.set_selected((fractional_values < 0), 0)
fractional_values.set_selected((fractional_values > 1), 1)
rmsd_est = 1.5 * flex.exp(4 * (0.7 - fractional_values))
return rmsd_est
def set_k_isotropic_exp(self, r_start,use_scale_r, verbose,
b_lower_limit = -100):
arrays = mmtbx.arrays.init(
f_calc = self.core.f_calc,
f_masks = self.core.f_mask(),
k_isotropic = self.core.k_isotropic,
k_anisotropic = self.core.k_anisotropic,
k_masks = self.core.k_mask())
sel = self.selection_work.data()
rf = scitbx.math.gaussian_fit_1d_analytical(
x = flex.sqrt(self.ss).select(sel),
y = self.f_obs.data().select(sel),
z = abs(arrays.f_model).data().select(sel))
if(rf.b < b_lower_limit): return r_start
k = rf.a * flex.exp(-self.ss * rf.b)
r = self.try_scale(k_isotropic_exp = k, use_scale_r=use_scale_r)
if(r<r_start):
if(verbose):
print >> self.log, " r(set_k_isotropic_exp): %6.4f"%r
self.core = self.core.update(k_isotropic_exp = k)
r_start = self.r_factor()
else:
if(verbose):
print >> self.log, " r(set_k_isotropic_exp): %6.4f (rejected)"%r
return r_start
def set_k_isotropic_exp(self,
r_start,
use_scale_r,
verbose,
b_lower_limit=-100):
arrays = mmtbx.arrays.init(f_calc=self.core.f_calc,
f_masks=self.core.f_mask(),
k_isotropic=self.core.k_isotropic,
k_anisotropic=self.core.k_anisotropic,
k_masks=self.core.k_mask())
sel = self.selection_work.data()
rf = scitbx.math.gaussian_fit_1d_analytical(
x=flex.sqrt(self.ss).select(sel),
y=self.f_obs.data().select(sel),
z=abs(arrays.f_model).data().select(sel))
if (rf.b < b_lower_limit): return r_start
k = rf.a * flex.exp(-self.ss * rf.b)
r = self.try_scale(k_isotropic_exp=k, use_scale_r=use_scale_r)
if (r < r_start):
if (verbose):
print >> self.log, " r(set_k_isotropic_exp): %6.4f" % r
self.core = self.core.update(k_isotropic_exp=k)
r_start = self.r_factor()
else:
if (verbose):
print >> self.log, " r(set_k_isotropic_exp): %6.4f (rejected)" % r
return r_start
def __init__(self, d_i, psi_i, eta_rad, Deff):
import sys
self.safelog = -1. + math.log(sys.float_info.max)
self.S = StringIO.StringIO()
pickle.dump([d_i, psi_i, eta_rad, Deff],self.S,0)
assert len(d_i) == len(psi_i)
self.d_i = d_i
self.psi_i = psi_i
self.Nobs = len(d_i)
self.escalate = 10. # 10 is a soft switch; 50-100 a hard switch
self.x = flex.double([log(2./Deff), log(eta_rad)]) # parameters alpha, eta
self.minimizer = run(
target_evaluator=self,
core_params=core_parameters(
gtol=0.1
# increasing the accuracy of the line search technique (default=0.9)
# as suggested by source code. Otherwise Deff is set unreasonably high
# and the exponential blows up.
),
termination_params = termination_parameters(
traditional_convergence_test=False,
drop_convergence_test_max_drop_eps=1.e-5,
min_iterations=0,
max_iterations = 100,
max_calls=200),
exception_handling_params=exception_handling_parameters(
ignore_line_search_failed_rounding_errors=True,
ignore_line_search_failed_step_at_lower_bound=True,#the only change from default
ignore_line_search_failed_step_at_upper_bound=False,
ignore_line_search_failed_maxfev=False,
ignore_line_search_failed_xtol=False,
ignore_search_direction_not_descent=False)
)
self.x=flex.exp(self.x)
def target(self, combined_vector):
vector = []
for ii in range(self.n_refine):
vector.append(combined_vector[ii * self.n:(ii + 1) * self.n])
if (self.both):
for ii in range(self.n_refine):
self.rbe.rotate_translate(vector[ii][0:3], vector[ii][3:],
ii + 1)
elif (self.translate):
for ii in range(self.n_refine):
self.rbe.rotate_translate(vector[ii], self.angle, ii + 1)
else:
for ii in range(self.n_refine):
self.rbe.rotate_translate(self.vector, vector[ii], ii + 1)
if (self.data_type == 'pr'):
calc_pr = self.rbe.get_norm_pr()
score = 1000 * flex.sum(
flex.abs(calc_pr - self.data) *
flex.exp(-self.data * self.data))
clash = self.rbe.get_clash()
score += clash / score
else:
calc_i = self.rbe.get_intensity()
scale = flex.sum(calc_i * self.data.i) / flex.sum(
flex.pow2(calc_i))
score = flex.sum(flex.abs(scale * calc_i - self.data.i))
return score
def q_range_analyses(self, rg, io, rat_lim=1.5, window_size=3, level=10.0, sigma=False):
selector = flex.bool(self.data.q*rg<rat_lim)
tmp_q = self.data.q.select( selector )
tmp_i = self.data.i.select( selector )
tmp_s = self.data.s.select( selector )
rg2 = rg*rg
lni = math.log( io )
cs = None
if sigma:
cs = self.chi_square( lni, rg2, tmp_q, tmp_i, tmp_s,False )
cs = flex.sqrt( cs )
else:
cs = flex.exp(lni-rg2*tmp_q*tmp_q/3.0)
ss = cs/100.0
cs = flex.abs(tmp_i-cs)/ss
not_okai_ranges = []
previous_one_was_bad=False
tmp_range = []
for ii in xrange( window_size, cs.size() ):
tmp_cs = flex.mean( cs[ii-window_size:ii] )
if tmp_cs > level:
if not previous_one_was_bad:
tmp_range.append( tmp_q[ii] )
tmp_range.append( tmp_q[ii] )
previous_one_was_bad=True
else:
tmp_range[1] = tmp_q[ii]
else:
previous_one_was_bad=False
if len(tmp_range)>0:
not_okai_ranges.append( tmp_range )
tmp_range=[]
return not_okai_ranges
def test_curve_scaler():
flex.set_random_seed(12345)
x = flex.double(range(10)) / 10.0
y = flex.exp(-x)
y0 = y
y1 = y * 100 + 50
y2 = y * 1000 + 500
v0 = y0 * 0 + 1
v1 = y0 * 0 + 1.0
v2 = v1
scaler_object = linear_scaler([y0, y1, y2], [v0, v1, v2],
0,
factor=5,
show_progress=False)
s, o, m = scaler_object.retrieve_results()
assert approx_equal(s[1], 0.01, eps=1e-3)
assert approx_equal(s[2], 0.001, eps=1e-3)
assert approx_equal(o[1], -50, eps=1e-2)
assert approx_equal(o[2], -500, eps=1e-2)
s, o = scale_it_pairwise([y0, y1, y2], [v0, v1, v2],
0,
show_progress=False)
assert approx_equal(s[1], 0.01, eps=1e-3)
assert approx_equal(s[2], 0.001, eps=1e-3)
assert approx_equal(o[1], -50, eps=1e-2)
assert approx_equal(o[2], -500, eps=1e-2)
def __init__(self, d_pos, psi_pos, d_neg, psi_neg, eta_rad, Deff):
adopt_init_args(self, locals())
import sys
self.safelog = -1. + math.log(sys.float_info.max)
assert len(d_pos) == len(psi_pos)
assert len(d_neg) == len(psi_neg)
self.Nobs = len(d_pos) + len(d_neg)
self.escalate = 10. # 10 is a soft switch; 50-100 a hard switch
self.x = flex.double([log(2. / Deff),
log(eta_rad)]) # parameters alpha, eta
self.minimizer = run(
target_evaluator=self,
core_params=core_parameters(
gtol=0.1
# increasing the accuracy of the line search technique (default=0.9)
# as suggested by source code. Otherwise Deff is set unreasonably high
# and the exponential blows up.
),
termination_params=termination_parameters(
traditional_convergence_test=False,
drop_convergence_test_max_drop_eps=1.e-5,
min_iterations=0,
max_iterations=100,
max_calls=200),
exception_handling_params=exception_handling_parameters(
ignore_line_search_failed_rounding_errors=True,
ignore_line_search_failed_step_at_lower_bound=
True, #the only change from default
ignore_line_search_failed_step_at_upper_bound=False,
ignore_line_search_failed_maxfev=False,
ignore_line_search_failed_xtol=False,
ignore_search_direction_not_descent=False))
self.x = flex.exp(self.x)
def __call__(self, x_obs):
shape, location, scale = self.params
normal_part = (2 / (scale * math.sqrt(2 * math.pi))
* flex.exp(- flex.pow2(x_obs - location)/(2 * scale**2)))
cdf_part = 0.5 * (
1 + scitbx.math.erf(shape * (x_obs - location)/ (math.sqrt(2) * scale)))
y_calc = normal_part * cdf_part
return y_calc
def test_log_inv_fit():
x = flex.double(range(0, 100)) * 0.01
p = (1, 2)
yo = flex.double(x.size())
for i in range(len(p)):
yo += 1 / flex.exp(p[i] * flex.pow(x, i))
yf = resolution_analysis.log_inv_fit(x, yo, degree=2)
assert yo == pytest.approx(yf, abs=1e-2)
def chi_square( self, lni, rg2, q,i,s, mean=True):
i_calc = flex.exp(lni-rg2*q*q/3.0)
d = (i-i_calc)/s
if mean:
d = flex.mean( d*d )
else:
d = d*d
return d
def __call__(self, x_obs):
shape, location, scale = self.params
normal_part = (2 / (scale * math.sqrt(2 * math.pi)) *
flex.exp(-flex.pow2(x_obs - location) / (2 * scale**2)))
cdf_part = 0.5 * (1 + scitbx.math.erf(shape * (x_obs - location) /
(math.sqrt(2) * scale)))
y_calc = normal_part * cdf_part
return y_calc
def build_up(self, objective_only=False):
x1, x2, x3, x4 = self.x
exp_x1_t = flex.exp(x1 * self.t)
exp_x2_t = flex.exp(x2 * self.t)
residuals = x3 * exp_x1_t + x4 * exp_x2_t
residuals -= self.y
self.reset()
if objective_only:
self.add_residuals(residuals, weights=None)
else:
grad_r = (self.t * x3 * exp_x1_t, self.t * x4 * exp_x2_t, exp_x1_t,
exp_x2_t)
jacobian = flex.double(flex.grid(self.n_data, self.n_parameters))
for j, der_r in enumerate(grad_r):
jacobian.matrix_paste_column_in_place(der_r, j)
self.add_equations(residuals, jacobian, weights=None)
def pyplot(self):
from matplotlib import pyplot
pyplot.plot(self.x_obs, self.y_obs)
pyplot.plot(self.x_obs, self.compute_y_calc())
for i in range(self.n_gaussians):
scale, mu, S = tuple(self.x[i*3:i*3+3])
y_calc = scale * flex.exp(-flex.pow2(self.x_obs-mu) * S**2)
pyplot.plot(self.x_obs, y_calc)
pyplot.show()
def pyplot(self):
from matplotlib import pyplot
pyplot.plot(self.x_obs, self.y_obs)
pyplot.plot(self.x_obs, self.compute_y_calc())
for i in range(self.n_gaussians):
scale, mu, S = tuple(self.x[i * 3:i * 3 + 3])
y_calc = scale * flex.exp(-flex.pow2(self.x_obs - mu) * S**2)
pyplot.plot(self.x_obs, y_calc)
pyplot.show()
def do_gaussian_fit(scale, mu, sigma):
start = mu - 6 * sigma
stop = mu + 6 * sigma
step = (stop - start) / 1000
x = flex.double(frange(start, stop, step))
y = scale * flex.exp(-flex.pow2(x - mu) / (2 * sigma**2))
fit = curve_fitting.single_gaussian_fit(x, y)
assert approx_equal(fit.a, scale, 1e-4)
assert approx_equal(fit.b, mu, eps=1e-4)
assert approx_equal(fit.c, sigma, eps=1e-4)
def do_gaussian_fit(scale, mu, sigma): start = mu - 6 * sigma stop = mu + 6 * sigma step = (stop - start)/1000 x = flex.double(frange(start, stop, step)) y = scale * flex.exp(-flex.pow2(x - mu) / (2 * sigma**2)) fit = curve_fitting.single_gaussian_fit(x, y) assert approx_equal(fit.a, scale, 1e-4) assert approx_equal(fit.b, mu, eps=1e-4) assert approx_equal(fit.c, sigma, eps=1e-4)
def build_up(self, objective_only=False):
x1, x2, x3, x4 = self.x
exp_x1_t = flex.exp(x1*self.t)
exp_x2_t = flex.exp(x2*self.t)
residuals = x3*exp_x1_t + x4*exp_x2_t
residuals -= self.y
self.reset()
if objective_only:
self.add_residuals(residuals, weights=None)
else:
grad_r = (self.t*x3*exp_x1_t,
self.t*x4*exp_x2_t,
exp_x1_t,
exp_x2_t)
jacobian = flex.double(flex.grid(self.n_data, self.n_parameters))
for j, der_r in enumerate(grad_r):
jacobian.matrix_paste_column_in_place(der_r, j)
self.add_equations(residuals, jacobian, weights=None)
def calc_new_intensity(self, q_array=None, rmax=None):
if rmax is None:
rmax = self.rmax
if q_array is None:
q_array = flex.double(range(51)) / 100.0
self.intensity = q_array * 0.0
self.intensity_vac = q_array * 0.0
self.intensity_sol = q_array * 0.0
self.intensity_lay = q_array * 0.0
scale_p = flex.exp(-0.23 * q_array * q_array)
scale_s = flex.exp(0.5 * q_array * q_array)
# Get the coefs #
self.zp_coef = self.zernike_mom_p.coefs() * self.protein
self.zs_coef = self.zernike_mom_s.coefs() * self.solvent
self.zb_coef = self.zernike_mom_b.coefs() * self.sol_layer
ii = 0 # starting with the first q
for qq, sp, ss in zip(q_array, scale_p, scale_s):
#Make a zernike model for particular q value qq #
this_z_model = zm.zernike_model(self.zernike_mom,
flex.double([qq]), rmax, self.nmax)
self.mom_update(sp, ss)
this_intensity = this_z_model.calc_intensity_nlm(self.zernike_mom)
this_intensity_vac = this_z_model.calc_intensity_nlm(
self.zernike_mom_p)
this_intensity_lay = this_z_model.calc_intensity_nlm(
self.zernike_mom_b)
this_intensity_sol = this_z_model.calc_intensity_nlm(
self.zernike_mom_s)
self.intensity_vac[ii] = (this_intensity_vac[0])
self.intensity[ii] = (this_intensity[0])
self.intensity_sol[ii] = (this_intensity_sol[0])
self.intensity_lay[ii] = (this_intensity_lay[0])
ii = ii + 1
## Normalized to Absolute I(0)
self.i_znk_0 = self.intensity_vac[0]
self.intensity_vac = self.intensity_vac * (self.abs_I0 /
self.intensity_vac[0])
self.intensity = self.intensity * (self.abs_I0 / self.i_znk_0)
self.intensity_lay = self.intensity_lay * (self.abs_I0 / self.i_znk_0)
self.intensity_sol = self.intensity_sol * (self.abs_I0 / self.i_znk_0)
return self.intensity, self.intensity_vac, self.intensity_sol, self.intensity_lay
def exercise_2():
x = flex.double()
z = flex.double()
for i in xrange(1,11):
x.append(i)
for i in xrange(11,21):
z.append(i)
y = z * 2 * flex.exp(-5.*x*x)
r = scitbx.math.gaussian_fit_1d_analytical(x = x, y = y, z = z)
assert approx_equal(r.a, 2., 1.e-6)
assert approx_equal(r.b, 5., 1.e-6)
def exercise_with_random_arguments(n_arguments, n_iterations):
mt = flex.mersenne_twister(seed=0)
d = mt.random_double(size=n_arguments) * 100 - 50
f = d.as_float()
print("showing wall clock times!")
t0 = time.time()
for i_iteration in range(n_iterations):
jef = scitbx.math.jacks_expf(array_of_float=f)
print("jacks_expf(): %.2f s" % (time.time() - t0))
for i_iteration in range(n_iterations):
sef = flex.exp(f)
print("std::exp(float): %.2f s" % (time.time() - t0))
t0 = time.time()
for i_iteration in range(n_iterations):
sed = flex.exp(d)
print("std::exp(double): %.2f s" % (time.time() - t0))
assert sed.all_gt(0)
for ef in [jef, sef]:
max_rel_err = flex.max(flex.abs((ef.as_double() - sed) / sed))
assert approx_equal(max_rel_err, 0, eps=1e-5)
def exercise_2():
x = flex.double()
z = flex.double()
for i in range(1, 11):
x.append(i)
for i in range(11, 21):
z.append(i)
y = z * 2 * flex.exp(-5. * x * x)
r = scitbx.math.gaussian_fit_1d_analytical(x=x, y=y, z=z)
assert approx_equal(r.a, 2., 1.e-6)
assert approx_equal(r.b, 5., 1.e-6)
def exercise_with_random_arguments(n_arguments, n_iterations):
mt = flex.mersenne_twister(seed=0)
d = mt.random_double(size=n_arguments) * 100 - 50
f = d.as_float()
print "showing wall clock times!"
t0 = time.time()
for i_iteration in xrange(n_iterations):
jef = scitbx.math.jacks_expf(array_of_float=f)
print "jacks_expf(): %.2f s" % (time.time() - t0)
for i_iteration in xrange(n_iterations):
sef = flex.exp(f)
print "std::exp(float): %.2f s" % (time.time() - t0)
t0 = time.time()
for i_iteration in xrange(n_iterations):
sed = flex.exp(d)
print "std::exp(double): %.2f s" % (time.time() - t0)
assert sed.all_gt(0)
for ef in [jef, sef]:
max_rel_err = flex.max(flex.abs((ef.as_double() - sed) / sed))
assert approx_equal(max_rel_err, 0, eps=1e-5)
def base(self, x, thres=600):
x_min = flex.min(x)
x_max = flex.max(x)
assert (x_min >= -1)
assert (x_max <= 1)
modi = self.polynome.f(x)
sel = flex.bool(modi > thres)
modi = modi.set_selected(sel, thres)
base_function = flex.exp(modi)
base_function = base_function * self.prior.base(x)
return base_function
def __init__(self, xray_structure, k_anisotropic, k_masks, ss):
self.xray_structure = xray_structure
self.k_anisotropic = k_anisotropic
self.k_masks = k_masks
self.ss = ss
#
k_total = self.k_anisotropic
r = scitbx.math.gaussian_fit_1d_analytical(x=flex.sqrt(self.ss),
y=k_total)
k, b = r.a, r.b
#
k, b, r = mmtbx.bulk_solvent.fit_k_exp_b_to_k_total(
k_total, self.ss, k, b)
k_exp_overall, b_exp_overall = None, None
if (r < 0.7): k_exp_overall, b_exp_overall = k, b
if (self.xray_structure is None): return None
b_adj = 0
if ([k_exp_overall, b_exp_overall].count(None) == 0 and k != 0):
bs1 = self.xray_structure.extract_u_iso_or_u_equiv(
) * adptbx.u_as_b(1.)
def split(b_trace, xray_structure):
b_min = xray_structure.min_u_cart_eigenvalue() * adptbx.u_as_b(
1.)
b_res = min(0, b_min + b_trace + 1.e-6)
b_adj = b_trace - b_res
xray_structure.shift_us(b_shift=b_adj)
return b_adj, b_res
b_adj, b_res = split(b_trace=b_exp_overall,
xray_structure=self.xray_structure)
k_new = k_exp_overall * flex.exp(-self.ss * b_adj)
bs2 = self.xray_structure.extract_u_iso_or_u_equiv(
) * adptbx.u_as_b(1.)
diff = bs2 - bs1
assert approx_equal(flex.min(diff), flex.max(diff))
assert approx_equal(flex.max(diff), b_adj)
self.k_anisotropic = self.k_anisotropic / k_new
self.k_masks = [
m * flex.exp(-self.ss * b_adj) for m in self.k_masks
]
def twod_integrator( cub, n_points ):
result = 0
x = flex.double()
y = flex.double()
w = flex.double()
for ii in xrange(n_points):
x.append( cub.coord(ii)[0] )
y.append( cub.coord(ii)[1] )
w.append( cub.weight(ii) )
tot = flex.exp( -x -y)*w
tot = flex.sum( tot )
return tot
def twod_integrator(cub, n_points):
result = 0
x = flex.double()
y = flex.double()
w = flex.double()
for ii in xrange(n_points):
x.append(cub.coord(ii)[0])
y.append(cub.coord(ii)[1])
w.append(cub.weight(ii))
tot = flex.exp(-x - y) * w
tot = flex.sum(tot)
return tot
def apply_back_trace_of_overall_exp_scale_matrix(self,
xray_structure=None):
k, b = self.overall_isotropic_kb_estimate()
k_total = self.core.k_isotropic * self.core.k_anisotropic * \
self.core.k_isotropic_exp
k, b, r = mmtbx.bulk_solvent.fit_k_exp_b_to_k_total(
k_total, self.ss, k, b)
if (r < 0.7): self.k_exp_overall, self.b_exp_overall = k, b
if (xray_structure is None): return None
b_adj = 0
if ([self.k_exp_overall, self.b_exp_overall].count(None) == 0
and k != 0):
bs1 = xray_structure.extract_u_iso_or_u_equiv() * adptbx.u_as_b(1.)
def split(b_trace, xray_structure):
b_min = xray_structure.min_u_cart_eigenvalue() * adptbx.u_as_b(
1.)
b_res = min(0, b_min + b_trace + 1.e-6)
b_adj = b_trace - b_res
xray_structure.shift_us(b_shift=b_adj)
return b_adj, b_res
b_adj, b_res = split(b_trace=self.b_exp_overall,
xray_structure=xray_structure)
k_new = self.k_exp_overall * flex.exp(-self.ss * b_adj)
bs2 = xray_structure.extract_u_iso_or_u_equiv() * adptbx.u_as_b(1.)
diff = bs2 - bs1
assert approx_equal(flex.min(diff), flex.max(diff))
assert approx_equal(flex.max(diff), b_adj)
self.core = self.core.update(
k_isotropic=self.core.k_isotropic,
k_isotropic_exp=self.core.k_isotropic_exp / k_new,
k_masks=[
m * flex.exp(-self.ss * b_adj) for m in self.core.k_masks
])
return group_args(xray_structure=xray_structure,
k_isotropic=self.k_isotropic(),
k_anisotropic=self.k_anisotropic(),
k_mask=self.k_masks(),
b_adj=b_adj)
def examples():
# an illustration of Hermite Gauss quadrature
# we will try to integrate
# Exp[-x^2-x] {x,-inf, inf}
# The true answer is Exp[0.25] Sqrt[Pi]
#
f_theory = math.exp(0.25)*math.sqrt( math.pi )
for ii in xrange(6,10):
ghq = sm.gauss_hermite_engine(ii)
w = ghq.w()
x = ghq.x()
f = flex.sum( (flex.exp( -x ))*w )
assert approx_equal(f,f_theory, eps=1e-5)
# an example of Gauss-Legendre integration
# we integrate exp(-x) between -1 and 1
f_theory = math.exp(1) - math.exp(-1)
for ii in xrange(5,90):
glq = sm.gauss_legendre_engine(ii)
w = glq.w()
x = glq.x()
f = flex.sum( (flex.exp( -x ))*w )
def weighted_means(self): min_score = flex.min( self.scores ) mw = flex.mean( self.scores ) self.scores = self.scores-min_score wghts = flex.exp( -self.scores*0.50 ) sw = 1e-12+flex.sum( wghts ) mrg = flex.sum(wghts*self.rg)/sw srg = flex.sum(wghts*self.rg*self.rg)/sw mi0 = flex.sum(wghts*self.i0)/sw si0 = flex.sum(wghts*self.i0*self.i0)/sw si0 = math.sqrt(si0-mi0*mi0) srg = math.sqrt(srg-mrg*mrg) return mrg,srg,mi0,si0,mw
def examples():
# an illustration of Hermite Gauss quadrature
# we will try to integrate
# Exp[-x^2-x] {x,-inf, inf}
# The true answer is Exp[0.25] Sqrt[Pi]
#
f_theory = math.exp(0.25) * math.sqrt(math.pi)
for ii in xrange(6, 10):
ghq = sm.gauss_hermite_engine(ii)
w = ghq.w()
x = ghq.x()
f = flex.sum((flex.exp(-x)) * w)
assert approx_equal(f, f_theory, eps=1e-5)
# an example of Gauss-Legendre integration
# we integrate exp(-x) between -1 and 1
f_theory = math.exp(1) - math.exp(-1)
for ii in xrange(5, 90):
glq = sm.gauss_legendre_engine(ii)
w = glq.w()
x = glq.x()
f = flex.sum((flex.exp(-x)) * w)
def rotate_image(self,data,angle_deg,sigma=1.0):
# we take an image, rotate, interpolate it back onto a regular grid and return it
#
# There is however the problem that some pixels will never be seen. those pixels will be filled up by interpolation
data = data*self.mask
cost = smath.cos( angle_deg*smath.pi/180.0 )
sint = smath.sin( angle_deg*smath.pi/180.0 )
new_image = flex.double(self.grid,0)
visited = flex.bool(self.grid,False)
nx_array = flex.double()
ny_array = flex.double()
val_array = flex.double()
for ix in range(self.np):
for iy in range(self.np):
x = ix - self.ox
y = iy - self.oy
nx = cost*x - sint*y
ny = sint*x + cost*y
jx = int(nx+self.ox+0.5)
jy = int(ny+self.oy+0.5)
if jx >= self.np : jx = self.np-1
if jy >= self.np : jy = self.np-1
if jx<0: jx=0
if jy<0: jy=0
new_image[ (jx,jy) ] = data[ (ix,iy) ]
visited[ (jx,jy) ] = True
nx_array.append( nx+self.ox )
ny_array.append( ny+self.oy )
val_array.append( data[ (ix,iy) ] )
assert nx_array.size() == self.mask.size()
not_visited = ~visited
not_visited = not_visited.iselection()
# now we need to loop over all non-visited pixel values
for pixel in not_visited:
#for ix in range(self.np):
# for iy in range(self.np):
nv = 0
if self.mask[(ix,iy)]>-0.1:
nv = -9
index = n_dim_index_from_one_dim(pixel, [self.np,self.np] )
nvx = index[1]
nvy = index[0]
dx = nx_array-nvx
dy = ny_array-nvy
dd = (dx*dx+dy*dy)
ss = flex.exp( -dd/(sigma*sigma) )*self.mask
nv = flex.sum(ss*val_array)/(1e-12+flex.sum(ss))
new_image[ (ix,iy) ] = nv
new_image = new_image*self.mask
return new_image
def k_mask_grid_search(self, r_start):
k_mask_trial_range = flex.double(
[i / 1000. for i in range(0, 650, 50)])
k_mask = flex.double(self.f_obs.size(), 0)
k_mask_bin = flex.double()
for i_cas, cas in enumerate(self.cores_and_selections):
selection, core, selection_use, sel_work = cas
scale = self.core.k_anisotropic.select(selection) * \
self.core.k_isotropic.select(selection) * \
self.core.k_isotropic_exp.select(selection)
f_obs = self.f_obs.select(selection)
k_mask_bin_, k_isotropic_bin_ = \
bulk_solvent.k_mask_and_k_overall_grid_search(
f_obs.data()/scale,
core.f_calc.data(),
core.f_mask().data(),
k_mask_trial_range,
selection_use)
k_mask_bin.append(k_mask_bin_)
k_mask.set_selected(selection, k_mask_bin_)
k_mask_bin_smooth = self.smooth(k_mask_bin)
k_mask = self.populate_bin_to_individual_k_mask_linear_interpolation(
k_mask_bin=k_mask_bin_smooth)
k_isotropic = self._k_isotropic_as_scale_k1(r_start=r_start,
k_mask=k_mask)
self.core = self.core.update(k_masks=k_mask, k_isotropic=k_isotropic)
self.bss_result.k_mask_bin_orig = k_mask_bin
self.bss_result.k_mask_bin_smooth = k_mask_bin_smooth
self.bss_result.k_mask = k_mask
self.bss_result.k_isotropic = k_isotropic
r_start = self.r_factor()
####
if (len(self.cores_and_selections) > 2):
x = flex.double()
y = flex.double()
for i_sel, cas in enumerate(self.cores_and_selections):
selection, core, selection_use, sel_work = cas
sel = sel_work
ss_ = self.ss_bin_values[i_sel][2]
k_isotropic_ = flex.mean(self.core.k_isotropic.select(sel))
x.append(ss_)
y.append(k_isotropic_)
import scitbx.math
r = scitbx.math.gaussian_fit_1d_analytical(x=flex.sqrt(x), y=y)
r_start = self.r_factor()
k_isotropic = r.a * flex.exp(-self.ss * r.b)
r = self.try_scale(k_isotropic=k_isotropic)
if (r < r_start):
self.core = self.core.update(k_isotropic=k_isotropic)
r_start = self.r_factor()
return r_start
def partial_derivatives(self, x_obs):
shape, location, scale = self.params
exponential_part = (1/(math.sqrt(2 * math.pi))
* flex.exp(- flex.pow2(x_obs - location)/(2 * scale**2)))
normal_part = 2 / scale * exponential_part
cdf_part = 0.5 * (
1 + scitbx.math.erf(shape * (x_obs - location)/ (math.sqrt(2) * scale)))
d_normal_part_d_location = 2 / scale**3 * (x_obs - location) * exponential_part
d_normal_part_d_scale = \
2 / scale**4 * (flex.pow2(x_obs - location) - scale**2) * exponential_part
exponential_part_with_shape = (
1 / (math.sqrt(math.pi)) *
flex.exp(-shape**2 * flex.pow2(x_obs - location)/(2 * scale**2)))
d_cdf_d_shape = \
(x_obs - location) / (math.sqrt(2) * scale) * exponential_part_with_shape
d_cdf_d_location = \
-shape / (math.sqrt(2) * scale) * exponential_part_with_shape
d_cdf_d_scale = (-shape * (x_obs - location) * exponential_part_with_shape /
(math.sqrt(2) * scale**2))
# product rule
return (d_cdf_d_shape * normal_part,
d_normal_part_d_location * cdf_part + d_cdf_d_location * normal_part,
d_normal_part_d_scale * cdf_part + d_cdf_d_scale * normal_part)
def get_z_scores(self, scale, b_value): i_scaled = flex.exp( self.calc_d_star_sq*b_value )*self.mean_calc*scale sel = ((self.mean_obs > 0) & (i_scaled > 0)) .iselection() ratio = self.mean_obs.select(sel) / i_scaled.select(sel) mean = self.curve( self.calc_d_star_sq ).select(sel) assert ratio.all_gt(0) # FIXME need to filter first! ratio = flex.log(ratio) var = self.std(self.calc_d_star_sq).select(sel) d_star_sq = self.calc_d_star_sq.select(sel) assert var.all_ne(0) z = flex.abs(ratio-mean)/var z_ = flex.double(self.mean_obs.size(), -1) z_.set_selected(sel, z) return z_
def target(self,vector):
v=1.0
scale = abs(vector[0])
b_value = vector[1]
if b_value > 200.0:
b_value = 200.0
if b_value < -200.0:
b_value = -200.0
i_scaled = flex.exp( self.calc_d_star_sq*b_value )*self.mean_calc*scale
ratio = i_scaled / self.mean_obs
curve = self.curve( self.calc_d_star_sq )
result = ratio - flex.exp(curve)
if (flex.max(result) > math.sqrt(sys.float_info.max)) :
raise OverflowError("Result array exceeds floating-point limit.")
result = result*result
wmax = flex.max(self.weight_array)
assert (wmax != 0)
if (wmax > 1) and (flex.max(result) > sys.float_info.max / wmax) :
raise OverflowError("Weighted result array will exceed floating-point "+
"limit: %e" % flex.max(result))
result = result*self.weight_array
result = flex.sum( result )
return result
def k_mask_grid_search(self, r_start):
k_mask_trial_range = flex.double([i/1000. for i in range(0,650,50)])
k_mask = flex.double(self.f_obs.size(), 0)
k_mask_bin = flex.double()
for i_cas, cas in enumerate(self.cores_and_selections):
selection, core, selection_use, sel_work = cas
scale = self.core.k_anisotropic.select(selection) * \
self.core.k_isotropic.select(selection) * \
self.core.k_isotropic_exp.select(selection)
f_obs = self.f_obs.select(selection)
k_mask_bin_, k_isotropic_bin_ = \
bulk_solvent.k_mask_and_k_overall_grid_search(
f_obs.data()/scale,
core.f_calc.data(),
core.f_mask().data(),
k_mask_trial_range,
selection_use)
k_mask_bin.append(k_mask_bin_)
k_mask.set_selected(selection, k_mask_bin_)
k_mask_bin_smooth = self.smooth(k_mask_bin)
k_mask = self.populate_bin_to_individual_k_mask_linear_interpolation(
k_mask_bin = k_mask_bin_smooth)
k_isotropic = self._k_isotropic_as_scale_k1(r_start=r_start, k_mask=k_mask)
self.core = self.core.update(k_masks = k_mask, k_isotropic = k_isotropic)
self.bss_result.k_mask_bin_orig = k_mask_bin
self.bss_result.k_mask_bin_smooth = k_mask_bin_smooth
self.bss_result.k_mask = k_mask
self.bss_result.k_isotropic = k_isotropic
r_start = self.r_factor()
####
if(len(self.cores_and_selections)>2):
x=flex.double()
y=flex.double()
for i_sel, cas in enumerate(self.cores_and_selections):
selection, core, selection_use, sel_work = cas
sel = sel_work
ss_ = self.ss_bin_values[i_sel][2]
k_isotropic_ = flex.mean(self.core.k_isotropic.select(sel))
x.append(ss_)
y.append(k_isotropic_)
import scitbx.math
r = scitbx.math.gaussian_fit_1d_analytical(x = flex.sqrt(x), y = y)
r_start = self.r_factor()
k_isotropic = r.a*flex.exp(-self.ss*r.b)
r = self.try_scale(k_isotropic = k_isotropic)
if(r<r_start):
self.core = self.core.update(k_isotropic = k_isotropic)
r_start = self.r_factor()
return r_start
def summary (self) :
i_scaled = flex.exp( self.calc_d_star_sq*self.b_value ) * \
self.mean_calc * self.scale
sel = (self.mean_obs > 0).iselection()
ratio = flex.log(i_scaled.select(sel) / self.mean_obs.select(sel))
ratio_ = flex.double(self.mean_obs.size(), 0)
ratio_.set_selected(sel, ratio)
curves = [
self.calc_d_star_sq,
-ratio_, # observed
self.curve( self.calc_d_star_sq ), # expected
self.get_z_scores(self.scale, self.b_value)
]
return summary(
all_curves=curves,
level=self.level,
all_bad_z_scores=self.all_bad_z_scores)
def test_curve_scaler(): flex.set_random_seed( 12345 ) x = flex.double( range(10) )/10.0 y = flex.exp( -x ) y0 = y y1 = y*100+50 y2 = y*1000+500 v0 = y0*0+1 v1 = y0*0+1.0 v2 = v1 scaler_object = linear_scaler( [y0,y1,y2], [v0,v1,v2], 0 , factor=5,show_progress=False) s,o,m = scaler_object.retrieve_results() assert approx_equal(s[1],0.01,eps=1e-3) assert approx_equal(s[2],0.001,eps=1e-3) assert approx_equal(o[1],-50,eps=1e-2) assert approx_equal(o[2],-500,eps=1e-2) s,o = scale_it_pairwise([y0,y1,y2],[v0,v1,v2],0, show_progress=False) assert approx_equal(s[1],0.01,eps=1e-3) assert approx_equal(s[2],0.001,eps=1e-3) assert approx_equal(o[1],-50,eps=1e-2) assert approx_equal(o[2],-500,eps=1e-2)
def partial_derivatives(self, x_obs):
a, b, c = self.params
exponential_part = flex.exp(-flex.pow2(x_obs - b) / (2 * c**2))
return(exponential_part,
a * (x_obs - b) / c**2 * exponential_part,
a * flex.pow2(x_obs - b) / c**3 * exponential_part)
def common_mode(self, img, stddev, mask):
"""The common_mode() function returns the mode of image stored in
the array pointed to by @p img. @p mask must be such that the @p
stddev at the selected pixels is greater than zero.
@param img 2D integer array of the image
@param stddev 2D integer array of the standard deviation of each
pixel in @p img
@param mask 2D Boolean array, @c True if the pixel is to be
included, @c False otherwise
@return Mode of the image, as a real number
"""
# Flatten the image and take out inactive pixels XXX because we
# cannot take means and medians of 2D arrays?
img_1d = img.as_1d().select(mask.as_1d()).as_double()
assert img_1d.size() > 0
if (self.common_mode_correction == "mean"):
# The common mode is approximated by the mean of the pixels with
# signal-to-noise ratio less than a given threshold. XXX Breaks
# if the selection is empty!
THRESHOLD_SNR = 2
img_snr = img_1d / stddev.as_double().as_1d().select(mask.as_1d())
return (flex.mean(img_1d.select(img_snr < THRESHOLD_SNR)))
elif (self.common_mode_correction == "median"):
return (flex.median(img_1d))
# Identify the common-mode correction as the peak histogram of the
# histogram of pixel values (the "standard" common-mode correction, as
# previously implemented in this class).
hist_min = -40
hist_max = 40
n_slots = 100
hist = flex.histogram(img_1d, hist_min, hist_max, n_slots=n_slots)
slots = hist.slots()
i = flex.max_index(slots)
common_mode = list(hist.slot_infos())[i].center()
if (self.common_mode_correction == "mode"):
return (common_mode)
# Determine the common-mode correction from the peak of a single
# Gaussian function fitted to the histogram.
from scitbx.math.curve_fitting import single_gaussian_fit
x = hist.slot_centers()
y = slots.as_double()
fit = single_gaussian_fit(x, y)
scale, mu, sigma = fit.a, fit.b, fit.c
self.logger.debug("fitted gaussian: mu=%.3f, sigma=%.3f" %(mu, sigma))
mode = common_mode
common_mode = mu
if abs(mode-common_mode) > 1000: common_mode = mode # XXX
self.logger.debug("delta common mode corrections: %.3f" %(mode-common_mode))
if 0 and abs(mode-common_mode) > 0:
#if 0 and skew > 0.5:
# view histogram and fitted gaussian
from numpy import exp
from matplotlib import pyplot
x_all = x
n, bins, patches = pyplot.hist(section_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
y_all = scale * flex.exp(-flex.pow2(x_all-mu) / (2 * sigma**2))
scale = slots[flex.max_index(slots)]
y_all *= scale/flex.max(y_all)
pyplot.plot(x_all, y_all)
pyplot.show()
return (common_mode)
def plotit(fobs,
sigma,
fcalc,
alpha,
beta,
epsilon,
centric,
out,
limit=5.0,
steps=1000,
plot_title="Outlier plot"):
fobs_a = flex.double( [fobs] )
fcalc_a = flex.double( [fcalc] )
epsilon_a = flex.double( [epsilon] )
alpha_a = flex.double( [alpha] )
beta_a = flex.double( [beta] )
centric_a = flex.bool ( [centric] )
p_calc = scaling.likelihood_ratio_outlier_test(
fobs_a,
None,
fcalc_a,
epsilon_a,
centric_a,
alpha_a,
beta_a)
print >> out
print >> out,"#Input parameters: "
print >> out,"#Title : ", plot_title
print >> out,"#F-calc : ", fcalc
print >> out,"#F-obs : ", fobs
print >> out,"#epsilon : ", epsilon
print >> out,"#alpha : ", alpha
print >> out,"#beta : ", beta
print >> out,"#centric : ", centric
mode = p_calc.posterior_mode()[0]
snd_der = math.sqrt(1.0/ math.fabs( p_calc.posterior_mode_snd_der()[0] ) )
print >> out,"#A Gaussian approximation of the likelihood function"
print >> out,"#could be constructed as follows with: "
print >> out,"# exp[-(fobs-mode)**2/(2*stdev**2)] /(sqrt(2 pi) stdev)"
print >> out,"#with"
print >> out,"#mode = ", mode
print >> out,"#stdev = ", snd_der
print >> out
print >> out,"#The log likelihood values for the mode and "
print >> out,"#observed values are"
print >> out,"#Log[P(fobs)] : ", p_calc.log_likelihood()[0]
print >> out,"#Log[P(mode)] : ", p_calc.posterior_mode_log_likelihood()[0]
print >> out,"#Their difference is:"
print >> out,"#delta : ", p_calc.log_likelihood()[0]-p_calc.posterior_mode_log_likelihood()[0]
print >> out,"#"
mean_fobs = p_calc.mean_fobs()
print >> out,"#mean f_obs : ", mean_fobs[0], " (first moment)"
low_limit = mode-snd_der*limit
if low_limit<0:
low_limit=0
high_limit = mode+limit*snd_der
if fobs < low_limit:
low_limit = fobs-2.0*snd_der
if low_limit<0:
low_limit=0
if fobs > high_limit:
high_limit = fobs+2.0*snd_der
fobs_a = flex.double( range(steps) )*(
high_limit-low_limit)/float(steps)+low_limit
fcalc_a = flex.double( [fcalc]*steps )
epsilon_a = flex.double( [epsilon]*steps )
alpha_a = flex.double( [alpha]*steps )
beta_a = flex.double( [beta]*steps )
centric_a = flex.bool ( [centric]*steps )
p_calc = scaling.likelihood_ratio_outlier_test(
fobs_a,
None,
fcalc_a,
epsilon_a,
centric_a,
alpha_a,
beta_a)
ll = p_calc.log_likelihood() #-p_calc.posterior_mode_log_likelihood()
ll = flex.exp( ll )
if (sigma is None) or (sigma <=0 ):
sigma=fobs/30.0
obs_gauss = (fobs_a - fobs)/float(sigma)
obs_gauss = flex.exp( -obs_gauss*obs_gauss/2.0 ) /(
math.sqrt(2.0*math.pi*sigma*sigma))
max_ll = flex.max( ll )*1.10
truncate_mask = flex.bool( obs_gauss >= max_ll )
obs_gauss = obs_gauss.set_selected( truncate_mask, max_ll )
ccp4_loggraph_plot = data_plots.plot_data(
plot_title=plot_title,
x_label = 'Fobs',
y_label = 'P(Fobs)',
x_data = fobs_a,
y_data = ll,
y_legend = 'P(Fobs|Fcalc,alpha,beta)',
comments = 'Fobs=%5.2f, sigma=%5.2f, Fcalc=%5.2f'%(fobs,sigma,fcalc) )
ccp4_loggraph_plot.add_data(
y_data = obs_gauss,
y_legend = "P(Fobs|<Fobs>,sigma)"
)
data_plots.plot_data_loggraph(ccp4_loggraph_plot,out)
def __call__(self, x_obs): a, b, c = self.params y_calc = a * flex.exp(-flex.pow2(x_obs - b) / (2 * c**2)) return y_calc
def experimental_array(x, z, sigz):
dd = (z - x * x) * (z - x * x) / (2.0 * sigz * sigz)
sel = flex.bool(dd > 100)
dd.set_selected(sel, 100)
result = (1.0 / (math.sqrt(2.0 * math.pi) * sigz)) * flex.exp(-dd)
return result
def centric(x):
result = math.sqrt(2.0 / math.pi) * flex.exp(-x * x / 2.0)
return result
def acentric(x):
result = 2.0 * x * flex.exp(-x * x)
return result
def example():
x_obs = (flex.double(range(100))+1.0)/101.0
y_ideal = flex.sin(x_obs*6.0*3.1415) + flex.exp(x_obs)
y_obs = y_ideal + (flex.random_double(size=x_obs.size())-0.5)*0.5
w_obs = flex.double(x_obs.size(),1)
print "Trying to determine the best number of terms "
print " via cross validation techniques"
print
n_terms = chebyshev_lsq_fit.cross_validate_to_determine_number_of_terms(
x_obs,y_obs,w_obs,
min_terms=5 ,max_terms=20,
n_goes=20,n_free=20)
print "Fitting with", n_terms, "terms"
print
fit = chebyshev_lsq_fit.chebyshev_lsq_fit(n_terms,x_obs,y_obs)
print "Least Squares residual: %7.6f" %(fit.f)
print " R2-value : %7.6f" %(fit.f/flex.sum(y_obs*y_obs))
print
fit_funct = chebyshev_polynome(
n_terms, fit.low_limit, fit.high_limit, fit.coefs)
y_fitted = fit_funct.f(x_obs)
abs_deviation = flex.max(
flex.abs( (y_ideal- y_fitted) ) )
print "Maximum deviation between fitted and error free data:"
print " %4.3f" %(abs_deviation)
abs_deviation = flex.mean(
flex.abs( (y_ideal- y_fitted) ) )
print "Mean deviation between fitted and error free data:"
print " %4.3f" %(abs_deviation)
print
abs_deviation = flex.max(
flex.abs( (y_obs- y_fitted) ) )
print "Maximum deviation between fitted and observed data:"
print " %4.3f" %(abs_deviation)
abs_deviation = flex.mean(
flex.abs( (y_obs- y_fitted) ) )
print "Mean deviation between fitted and observed data:"
print " %4.3f" %(abs_deviation)
print
print "Showing 10 points"
print " x y_obs y_ideal y_fit"
for ii in range(10):
print "%6.3f %6.3f %6.3f %6.3f" \
%(x_obs[ii*9], y_obs[ii*9], y_ideal[ii*9], y_fitted[ii*9])
try:
from iotbx import data_plots
except ImportError:
pass
else:
print "Preparing output for loggraph in a file called"
print " chebyshev.loggraph"
chebyshev_plot = data_plots.plot_data(plot_title='Chebyshev fitting',
x_label = 'x values',
y_label = 'y values',
x_data = x_obs,
y_data = y_obs,
y_legend = 'Observed y values',
comments = 'Chebyshev fit')
chebyshev_plot.add_data(y_data=y_ideal,
y_legend='Error free y values')
chebyshev_plot.add_data(y_data=y_fitted,
y_legend='Fitted chebyshev approximation')
output_logfile=open('chebyshev.loggraph','w')
f = StringIO()
data_plots.plot_data_loggraph(chebyshev_plot,f)
output_logfile.write(f.getvalue())
def get_p_of_r(self,x): base = flex.pow((1.0-x*x),self.k) exp_pol = flex.exp( self.polynome.f( x ) ) result = exp_pol*base return result
def exercise_gaussian_fit():
# test fitting of a gaussian
def do_gaussian_fit(scale, mu, sigma):
start = mu - 6 * sigma
stop = mu + 6 * sigma
step = (stop - start)/1000
x = flex.double(frange(start, stop, step))
y = scale * flex.exp(-flex.pow2(x - mu) / (2 * sigma**2))
fit = curve_fitting.single_gaussian_fit(x, y)
assert approx_equal(fit.a, scale, 1e-4)
assert approx_equal(fit.b, mu, eps=1e-4)
assert approx_equal(fit.c, sigma, eps=1e-4)
for i in range(10):
scale = random.random() * 1000
sigma = (random.random() + 0.0001) * 10
mu = (-1)**random.randint(0,1) * random.random() * 1000
functor = curve_fitting.gaussian(scale, mu, sigma)
start = mu - 6 * sigma
stop = mu + 6 * sigma
step = (stop - start)/1000
x = flex.double(frange(start, stop, step))
fd_grads = finite_differences(functor, x)
assert approx_equal(functor.partial_derivatives(x), fd_grads, 1e-4)
do_gaussian_fit(scale, mu, sigma)
# if we take the log of a gaussian we can fit a parabola
scale = 123
mu = 3.2
sigma = 0.1
x = flex.double(frange(2, 4, 0.01))
y = scale * flex.exp(-flex.pow2(x - mu) / (2 * sigma**2))
# need to be careful to only use values of y > 0
eps = 1e-15
x = flex.double([x[i] for i in range(x.size()) if y[i] > eps])
y = flex.double([y[i] for i in range(y.size()) if y[i] > eps])
fit = curve_fitting.univariate_polynomial_fit(x, flex.log(y), degree=2)
c, b, a = fit.params
assert approx_equal(mu, -b/(2*a))
assert approx_equal(sigma*sigma, -1/(2*a))
# test multiple gaussian fits
gaussians = [curve_fitting.gaussian(0.3989538, 3.7499764, 0.7500268),
curve_fitting.gaussian(0.7978957, 6.0000004, 0.5000078)]
x = flex.double(frange(0, 10, 0.1))
y = flex.double(x.size())
for i in range(len(gaussians)):
g = gaussians[i]
scale, mu, sigma = g.a, g.b, g.c
y += g(x)
starting_gaussians = [
curve_fitting.gaussian(1, 4, 1),
curve_fitting.gaussian(1, 5, 1)]
fit = curve_fitting.gaussian_fit(x, y, starting_gaussians)
for g1, g2 in zip(gaussians, fit.gaussians):
assert approx_equal(g1.a, g2.a, eps=1e-4)
assert approx_equal(g1.b, g2.b, eps=1e-4)
assert approx_equal(g1.c, g2.c, eps=1e-4)
# use example of 5-gaussian fit from here:
# http://research.stowers-institute.org/efg/R/Statistics/MixturesOfDistributions/index.htm
gaussians = [curve_fitting.gaussian(0.10516252, 23.32727, 2.436638),
curve_fitting.gaussian(0.46462715, 33.09053, 2.997594),
curve_fitting.gaussian(0.29827916, 41.27244, 4.274585),
curve_fitting.gaussian(0.08986616, 51.24468, 5.077521),
curve_fitting.gaussian(0.04206501, 61.31818, 7.070303)]
x = flex.double(frange(0, 80, 0.1))
y = flex.double(x.size())
for i in range(len(gaussians)):
g = gaussians[i]
scale, mu, sigma = g.a, g.b, g.c
y += g(x)
termination_params = scitbx.lbfgs.termination_parameters(
min_iterations=500)
starting_gaussians = [curve_fitting.gaussian(1, 21, 2.1),
curve_fitting.gaussian(1, 30, 2.8),
curve_fitting.gaussian(1, 40, 2.2),
curve_fitting.gaussian(1, 51, 1.2),
curve_fitting.gaussian(1, 60, 2.3)]
fit = curve_fitting.gaussian_fit(
x, y, starting_gaussians, termination_params=termination_params)
y_calc = fit.compute_y_calc()
assert approx_equal(y, y_calc, eps=1e-2)
have_cma_es = libtbx.env.has_module("cma_es")
if have_cma_es:
fit = curve_fitting.cma_es_minimiser(starting_gaussians, x, y)
y_calc = fit.compute_y_calc()
assert approx_equal(y, y_calc, eps=5e-2)