def exercise_linear_normal_equations():
py_eqs = [ ( 1, (-1, 0, 0), 1),
( 2, ( 2, -1, 0), 3),
(-1, ( 0, 2, 1), 2),
(-2, ( 0, 1, 0), -2),
]
eqs_0 = normal_eqns.linear_ls(3)
for b, a, w in py_eqs:
eqs_0.add_equation(right_hand_side=b,
design_matrix_row=flex.double(a),
weight=w)
eqs_1 = normal_eqns.linear_ls(3)
b = flex.double()
w = flex.double()
a = sparse.matrix(len(py_eqs), 3)
for i, (b_, a_, w_) in enumerate(py_eqs):
b.append(b_)
w.append(w_)
for j in xrange(3):
if a_[j]: a[i, j] = a_[j]
eqs_1.add_equations(right_hand_side=b, design_matrix=a, weights=w)
assert approx_equal(
eqs_0.normal_matrix_packed_u(), eqs_1.normal_matrix_packed_u(), eps=1e-15)
assert approx_equal(
eqs_0.right_hand_side(), eqs_1.right_hand_side(), eps=1e-15)
assert approx_equal(
list(eqs_0.normal_matrix_packed_u()), [ 13, -6, 0, 9, 4, 2 ], eps=1e-15)
assert approx_equal(
list(eqs_0.right_hand_side()), [ 11, -6, -2 ], eps=1e-15)
def exercise_linear_normal_equations():
py_eqs = [
(1, (-1, 0, 0), 1),
(2, (2, -1, 0), 3),
(-1, (0, 2, 1), 2),
(-2, (0, 1, 0), -2),
]
eqs_0 = normal_eqns.linear_ls(3)
for b, a, w in py_eqs:
eqs_0.add_equation(right_hand_side=b,
design_matrix_row=flex.double(a),
weight=w)
eqs_1 = normal_eqns.linear_ls(3)
b = flex.double()
w = flex.double()
a = sparse.matrix(len(py_eqs), 3)
for i, (b_, a_, w_) in enumerate(py_eqs):
b.append(b_)
w.append(w_)
for j in xrange(3):
if a_[j]: a[i, j] = a_[j]
eqs_1.add_equations(right_hand_side=b, design_matrix=a, weights=w)
assert approx_equal(eqs_0.normal_matrix_packed_u(),
eqs_1.normal_matrix_packed_u(),
eps=1e-15)
assert approx_equal(eqs_0.right_hand_side(),
eqs_1.right_hand_side(),
eps=1e-15)
assert approx_equal(list(eqs_0.normal_matrix_packed_u()),
[13, -6, 0, 9, 4, 2],
eps=1e-15)
assert approx_equal(list(eqs_0.right_hand_side()), [11, -6, -2], eps=1e-15)
def mcd_finite_sample(p, n, alpha):
"""Finite sample correction factor for the MCD estimate. Described in
Pison et al. Metrika (2002). doi.org/10.1007/s001840200191. Implementation
based on 'rawcorfactor' in fastmcd.m from Continuous Sound and Vibration
Analysis by Edward Zechmann"""
from math import log, exp
from scitbx.lstbx import normal_eqns
if p > 2:
coeffqpkwad500=[[-1.42764571687802,1.26263336932151,2],
[-1.06141115981725,1.28907991440387,3]]
coeffqpkwad875=[[-0.455179464070565,1.11192541278794,2],
[-0.294241208320834,1.09649329149811,3]]
y_500 = [log(-coeffqpkwad500[0][0] / p**coeffqpkwad500[0][1]),
log(-coeffqpkwad500[1][0] / p**coeffqpkwad500[1][1])]
y_875 = [log(-coeffqpkwad875[0][0] / p**coeffqpkwad875[0][1]),
log(-coeffqpkwad875[1][0] / p**coeffqpkwad875[1][1])]
A_500 = [[1, -log(coeffqpkwad500[0][2]*p**2)],
[1, -log(coeffqpkwad500[1][2]*p**2)]]
A_875 = [[1, -log(coeffqpkwad875[0][2]*p**2)],
[1, -log(coeffqpkwad875[1][2]*p**2)]]
# solve the set of equations labelled _500
eqs = normal_eqns.linear_ls(2)
for i in range(2):
eqs.add_equation(right_hand_side=y_500[i],
design_matrix_row=flex.double(A_500[i]),
weight=1)
eqs.solve()
coeffic_500 = eqs.solution()
# solve the set of equations labelled _875
eqs = normal_eqns.linear_ls(2)
for i in range(2):
eqs.add_equation(right_hand_side=y_875[i],
design_matrix_row=flex.double(A_875[i]),
weight=1)
eqs.solve()
coeffic_875 = eqs.solution()
fp_500_n = 1 - exp(coeffic_500[0]) / n**coeffic_500[1]
fp_875_n = 1 - exp(coeffic_875[0]) / n**coeffic_875[1]
elif p == 2:
fp_500_n = 1 - exp(0.673292623522027) / n**0.691365864961895
fp_875_n = 1 - exp(0.446537815635445) / n**1.06690782995919
elif p == 1:
fp_500_n = 1 - exp(0.262024211897096) / n**0.604756680630497
fp_875_n = 1 - exp(-0.351584646688712) / n**1.01646567502486
if alpha <= 0.875:
fp_alpha_n = fp_500_n + (fp_875_n - fp_500_n)/0.375 * (alpha - 0.5)
if 0.875 < alpha and alpha <= 1:
fp_alpha_n = fp_875_n + (1 - fp_875_n)/0.125 * (alpha - 0.875)
return 1/fp_alpha_n
def linear_regression(x, y, fit_intercept=True, fit_gradient=True):
"""Perform linear regression by least squares to model how a response
variable (y) is linearly-related to an explanatory variable (x).
Args:
x: Sequence of values of the explanatory variable
y: Sequence of values of the response variable (observations)
fit_gradient (bool): Fit the gradient as a parameter of the model
(otherwise assume unit gradient).
fit_intercept (bool): Fit the intercept as a parameter of the model
(otherwise assume an intercept of zero).
Returns:
list: Values of the model parameters
"""
from scitbx import sparse
n_obs = len(y)
assert len(x) == n_obs
n_param = [fit_gradient, fit_intercept].count(True)
assert n_param > 0
eqns = normal_eqns.linear_ls(n_param)
# The method add_equations requires a sparse matrix, so must convert to that
# even though we build the design matrix as dense.
a = sparse.matrix(n_obs, n_param)
icol = 0
if fit_intercept:
col = flex.double(n_obs, 1)
col.reshape(flex.grid(n_obs, 1))
a.assign_block(col, 0, icol)
icol += 1
if fit_gradient:
col = flex.double(x)
col.reshape(flex.grid(n_obs, 1))
a.assign_block(col, 0, icol)
# Unweighted fit only, for now
w = flex.double(n_obs, 1)
eqns.add_equations(right_hand_side=y, design_matrix=a, weights=w)
eqns.solve()
return list(eqns.solution())
def derive_change_of_basis_op(from_hkl, to_hkl):
# exclude those reflections that we couldn't index
sel = (to_hkl != (0, 0, 0)) & (from_hkl != (0, 0, 0))
assert sel.count(True) >= 3 # need minimum of 3 equations ?
to_hkl = to_hkl.select(sel)
from_hkl = from_hkl.select(sel)
# for each miller index, solve a system of linear equations to find the
# change of basis operator
h, k, l = to_hkl.as_vec3_double().parts()
r = []
from scitbx.lstbx import normal_eqns
for i in range(3):
eqns = normal_eqns.linear_ls(3)
for index, hkl in zip((h, k, l)[i], from_hkl):
eqns.add_equation(
right_hand_side=index, design_matrix_row=flex.double(hkl), weight=1
)
eqns.solve()
r.extend(eqns.solution())
from scitbx import matrix
from scitbx.math import continued_fraction
denom = 12
r = [
int(denom * continued_fraction.from_real(r_, eps=1e-2).as_rational())
for r_ in r
]
r = matrix.sqr(r).transpose()
# print (1/denom)*r
# now convert into a cctbx change_of_basis_op object
change_of_basis_op = sgtbx.change_of_basis_op(
sgtbx.rt_mx(sgtbx.rot_mx(r, denominator=denom))
).inverse()
print("discovered change_of_basis_op=%s" % (str(change_of_basis_op)))
# sanity check that this is the right cb_op
assert (change_of_basis_op.apply(from_hkl) == to_hkl).count(False) == 0
return change_of_basis_op
def derive_change_of_basis_op(from_hkl, to_hkl):
# exclude those reflections that we couldn't index
sel = (to_hkl != (0,0,0)) & (from_hkl != (0,0,0))
assert sel.count(True) >= 3 # need minimum of 3 equations ?
to_hkl = to_hkl.select(sel)
from_hkl = from_hkl.select(sel)
# for each miller index, solve a system of linear equations to find the
# change of basis operator
h, k, l = to_hkl.as_vec3_double().parts()
r = []
from scitbx.lstbx import normal_eqns
for i in range(3):
eqns = normal_eqns.linear_ls(3)
for index, hkl in zip((h,k,l)[i], from_hkl):
eqns.add_equation(right_hand_side=index,
design_matrix_row=flex.double(hkl),
weight=1)
eqns.solve()
r.extend(eqns.solution())
from scitbx.math import continued_fraction
from scitbx import matrix
denom = 12
r = [int(denom * continued_fraction.from_real(r_, eps=1e-2).as_rational())
for r_ in r]
r = matrix.sqr(r).transpose()
#print (1/denom)*r
# now convert into a cctbx change_of_basis_op object
change_of_basis_op = sgtbx.change_of_basis_op(
sgtbx.rt_mx(sgtbx.rot_mx(r, denominator=denom))).inverse()
print "discovered change_of_basis_op=%s" %(str(change_of_basis_op))
# sanity check that this is the right cb_op
assert (change_of_basis_op.apply(from_hkl) == to_hkl).count(False) == 0
return change_of_basis_op
def mcd_finite_sample(p, n, alpha):
"""Finite sample correction factor for the MCD estimate. Described in
Pison et al. Metrika (2002). doi.org/10.1007/s001840200191. Implementation
based on 'rawcorfactor' in fastmcd.m from Continuous Sound and Vibration
Analysis by Edward Zechmann"""
from scitbx.lstbx import normal_eqns
if p > 2:
coeffqpkwad500 = [
[-1.42764571687802, 1.26263336932151, 2],
[-1.06141115981725, 1.28907991440387, 3],
]
coeffqpkwad875 = [
[-0.455179464070565, 1.11192541278794, 2],
[-0.294241208320834, 1.09649329149811, 3],
]
y_500 = [
math.log(-coeffqpkwad500[0][0] / p ** coeffqpkwad500[0][1]),
math.log(-coeffqpkwad500[1][0] / p ** coeffqpkwad500[1][1]),
]
y_875 = [
math.log(-coeffqpkwad875[0][0] / p ** coeffqpkwad875[0][1]),
math.log(-coeffqpkwad875[1][0] / p ** coeffqpkwad875[1][1]),
]
A_500 = [
[1, -math.log(coeffqpkwad500[0][2] * p**2)],
[1, -math.log(coeffqpkwad500[1][2] * p**2)],
]
A_875 = [
[1, -math.log(coeffqpkwad875[0][2] * p**2)],
[1, -math.log(coeffqpkwad875[1][2] * p**2)],
]
# solve the set of equations labelled _500
eqs = normal_eqns.linear_ls(2)
for i in range(2):
eqs.add_equation(
right_hand_side=y_500[i],
design_matrix_row=flex.double(A_500[i]),
weight=1,
)
eqs.solve()
coeffic_500 = eqs.solution()
# solve the set of equations labelled _875
eqs = normal_eqns.linear_ls(2)
for i in range(2):
eqs.add_equation(
right_hand_side=y_875[i],
design_matrix_row=flex.double(A_875[i]),
weight=1,
)
eqs.solve()
coeffic_875 = eqs.solution()
fp_500_n = 1 - math.exp(coeffic_500[0]) / n ** coeffic_500[1]
fp_875_n = 1 - math.exp(coeffic_875[0]) / n ** coeffic_875[1]
elif p == 2:
fp_500_n = 1 - math.exp(0.673292623522027) / n**0.691365864961895
fp_875_n = 1 - math.exp(0.446537815635445) / n**1.06690782995919
elif p == 1:
fp_500_n = 1 - math.exp(0.262024211897096) / n**0.604756680630497
fp_875_n = 1 - math.exp(-0.351584646688712) / n**1.01646567502486
if alpha <= 0.875:
fp_alpha_n = fp_500_n + (fp_875_n - fp_500_n) / 0.375 * (alpha - 0.5)
if 0.875 < alpha and alpha <= 1:
fp_alpha_n = fp_875_n + (1 - fp_875_n) / 0.125 * (alpha - 0.875)
return 1 / fp_alpha_n