def exercise_linear_normal_equations(): py_eqs = [ ( 1, (-1, 0, 0), 1), ( 2, ( 2, -1, 0), 3), (-1, ( 0, 2, 1), 2), (-2, ( 0, 1, 0), -2), ] eqs_0 = normal_eqns.linear_ls(3) for b, a, w in py_eqs: eqs_0.add_equation(right_hand_side=b, design_matrix_row=flex.double(a), weight=w) eqs_1 = normal_eqns.linear_ls(3) b = flex.double() w = flex.double() a = sparse.matrix(len(py_eqs), 3) for i, (b_, a_, w_) in enumerate(py_eqs): b.append(b_) w.append(w_) for j in xrange(3): if a_[j]: a[i, j] = a_[j] eqs_1.add_equations(right_hand_side=b, design_matrix=a, weights=w) assert approx_equal( eqs_0.normal_matrix_packed_u(), eqs_1.normal_matrix_packed_u(), eps=1e-15) assert approx_equal( eqs_0.right_hand_side(), eqs_1.right_hand_side(), eps=1e-15) assert approx_equal( list(eqs_0.normal_matrix_packed_u()), [ 13, -6, 0, 9, 4, 2 ], eps=1e-15) assert approx_equal( list(eqs_0.right_hand_side()), [ 11, -6, -2 ], eps=1e-15)
def exercise_linear_normal_equations(): py_eqs = [ (1, (-1, 0, 0), 1), (2, (2, -1, 0), 3), (-1, (0, 2, 1), 2), (-2, (0, 1, 0), -2), ] eqs_0 = normal_eqns.linear_ls(3) for b, a, w in py_eqs: eqs_0.add_equation(right_hand_side=b, design_matrix_row=flex.double(a), weight=w) eqs_1 = normal_eqns.linear_ls(3) b = flex.double() w = flex.double() a = sparse.matrix(len(py_eqs), 3) for i, (b_, a_, w_) in enumerate(py_eqs): b.append(b_) w.append(w_) for j in xrange(3): if a_[j]: a[i, j] = a_[j] eqs_1.add_equations(right_hand_side=b, design_matrix=a, weights=w) assert approx_equal(eqs_0.normal_matrix_packed_u(), eqs_1.normal_matrix_packed_u(), eps=1e-15) assert approx_equal(eqs_0.right_hand_side(), eqs_1.right_hand_side(), eps=1e-15) assert approx_equal(list(eqs_0.normal_matrix_packed_u()), [13, -6, 0, 9, 4, 2], eps=1e-15) assert approx_equal(list(eqs_0.right_hand_side()), [11, -6, -2], eps=1e-15)
def mcd_finite_sample(p, n, alpha): """Finite sample correction factor for the MCD estimate. Described in Pison et al. Metrika (2002). doi.org/10.1007/s001840200191. Implementation based on 'rawcorfactor' in fastmcd.m from Continuous Sound and Vibration Analysis by Edward Zechmann""" from math import log, exp from scitbx.lstbx import normal_eqns if p > 2: coeffqpkwad500=[[-1.42764571687802,1.26263336932151,2], [-1.06141115981725,1.28907991440387,3]] coeffqpkwad875=[[-0.455179464070565,1.11192541278794,2], [-0.294241208320834,1.09649329149811,3]] y_500 = [log(-coeffqpkwad500[0][0] / p**coeffqpkwad500[0][1]), log(-coeffqpkwad500[1][0] / p**coeffqpkwad500[1][1])] y_875 = [log(-coeffqpkwad875[0][0] / p**coeffqpkwad875[0][1]), log(-coeffqpkwad875[1][0] / p**coeffqpkwad875[1][1])] A_500 = [[1, -log(coeffqpkwad500[0][2]*p**2)], [1, -log(coeffqpkwad500[1][2]*p**2)]] A_875 = [[1, -log(coeffqpkwad875[0][2]*p**2)], [1, -log(coeffqpkwad875[1][2]*p**2)]] # solve the set of equations labelled _500 eqs = normal_eqns.linear_ls(2) for i in range(2): eqs.add_equation(right_hand_side=y_500[i], design_matrix_row=flex.double(A_500[i]), weight=1) eqs.solve() coeffic_500 = eqs.solution() # solve the set of equations labelled _875 eqs = normal_eqns.linear_ls(2) for i in range(2): eqs.add_equation(right_hand_side=y_875[i], design_matrix_row=flex.double(A_875[i]), weight=1) eqs.solve() coeffic_875 = eqs.solution() fp_500_n = 1 - exp(coeffic_500[0]) / n**coeffic_500[1] fp_875_n = 1 - exp(coeffic_875[0]) / n**coeffic_875[1] elif p == 2: fp_500_n = 1 - exp(0.673292623522027) / n**0.691365864961895 fp_875_n = 1 - exp(0.446537815635445) / n**1.06690782995919 elif p == 1: fp_500_n = 1 - exp(0.262024211897096) / n**0.604756680630497 fp_875_n = 1 - exp(-0.351584646688712) / n**1.01646567502486 if alpha <= 0.875: fp_alpha_n = fp_500_n + (fp_875_n - fp_500_n)/0.375 * (alpha - 0.5) if 0.875 < alpha and alpha <= 1: fp_alpha_n = fp_875_n + (1 - fp_875_n)/0.125 * (alpha - 0.875) return 1/fp_alpha_n
def linear_regression(x, y, fit_intercept=True, fit_gradient=True): """Perform linear regression by least squares to model how a response variable (y) is linearly-related to an explanatory variable (x). Args: x: Sequence of values of the explanatory variable y: Sequence of values of the response variable (observations) fit_gradient (bool): Fit the gradient as a parameter of the model (otherwise assume unit gradient). fit_intercept (bool): Fit the intercept as a parameter of the model (otherwise assume an intercept of zero). Returns: list: Values of the model parameters """ from scitbx import sparse n_obs = len(y) assert len(x) == n_obs n_param = [fit_gradient, fit_intercept].count(True) assert n_param > 0 eqns = normal_eqns.linear_ls(n_param) # The method add_equations requires a sparse matrix, so must convert to that # even though we build the design matrix as dense. a = sparse.matrix(n_obs, n_param) icol = 0 if fit_intercept: col = flex.double(n_obs, 1) col.reshape(flex.grid(n_obs, 1)) a.assign_block(col, 0, icol) icol += 1 if fit_gradient: col = flex.double(x) col.reshape(flex.grid(n_obs, 1)) a.assign_block(col, 0, icol) # Unweighted fit only, for now w = flex.double(n_obs, 1) eqns.add_equations(right_hand_side=y, design_matrix=a, weights=w) eqns.solve() return list(eqns.solution())
def derive_change_of_basis_op(from_hkl, to_hkl): # exclude those reflections that we couldn't index sel = (to_hkl != (0, 0, 0)) & (from_hkl != (0, 0, 0)) assert sel.count(True) >= 3 # need minimum of 3 equations ? to_hkl = to_hkl.select(sel) from_hkl = from_hkl.select(sel) # for each miller index, solve a system of linear equations to find the # change of basis operator h, k, l = to_hkl.as_vec3_double().parts() r = [] from scitbx.lstbx import normal_eqns for i in range(3): eqns = normal_eqns.linear_ls(3) for index, hkl in zip((h, k, l)[i], from_hkl): eqns.add_equation( right_hand_side=index, design_matrix_row=flex.double(hkl), weight=1 ) eqns.solve() r.extend(eqns.solution()) from scitbx import matrix from scitbx.math import continued_fraction denom = 12 r = [ int(denom * continued_fraction.from_real(r_, eps=1e-2).as_rational()) for r_ in r ] r = matrix.sqr(r).transpose() # print (1/denom)*r # now convert into a cctbx change_of_basis_op object change_of_basis_op = sgtbx.change_of_basis_op( sgtbx.rt_mx(sgtbx.rot_mx(r, denominator=denom)) ).inverse() print("discovered change_of_basis_op=%s" % (str(change_of_basis_op))) # sanity check that this is the right cb_op assert (change_of_basis_op.apply(from_hkl) == to_hkl).count(False) == 0 return change_of_basis_op
def derive_change_of_basis_op(from_hkl, to_hkl): # exclude those reflections that we couldn't index sel = (to_hkl != (0,0,0)) & (from_hkl != (0,0,0)) assert sel.count(True) >= 3 # need minimum of 3 equations ? to_hkl = to_hkl.select(sel) from_hkl = from_hkl.select(sel) # for each miller index, solve a system of linear equations to find the # change of basis operator h, k, l = to_hkl.as_vec3_double().parts() r = [] from scitbx.lstbx import normal_eqns for i in range(3): eqns = normal_eqns.linear_ls(3) for index, hkl in zip((h,k,l)[i], from_hkl): eqns.add_equation(right_hand_side=index, design_matrix_row=flex.double(hkl), weight=1) eqns.solve() r.extend(eqns.solution()) from scitbx.math import continued_fraction from scitbx import matrix denom = 12 r = [int(denom * continued_fraction.from_real(r_, eps=1e-2).as_rational()) for r_ in r] r = matrix.sqr(r).transpose() #print (1/denom)*r # now convert into a cctbx change_of_basis_op object change_of_basis_op = sgtbx.change_of_basis_op( sgtbx.rt_mx(sgtbx.rot_mx(r, denominator=denom))).inverse() print "discovered change_of_basis_op=%s" %(str(change_of_basis_op)) # sanity check that this is the right cb_op assert (change_of_basis_op.apply(from_hkl) == to_hkl).count(False) == 0 return change_of_basis_op
def mcd_finite_sample(p, n, alpha): """Finite sample correction factor for the MCD estimate. Described in Pison et al. Metrika (2002). doi.org/10.1007/s001840200191. Implementation based on 'rawcorfactor' in fastmcd.m from Continuous Sound and Vibration Analysis by Edward Zechmann""" from scitbx.lstbx import normal_eqns if p > 2: coeffqpkwad500 = [ [-1.42764571687802, 1.26263336932151, 2], [-1.06141115981725, 1.28907991440387, 3], ] coeffqpkwad875 = [ [-0.455179464070565, 1.11192541278794, 2], [-0.294241208320834, 1.09649329149811, 3], ] y_500 = [ math.log(-coeffqpkwad500[0][0] / p ** coeffqpkwad500[0][1]), math.log(-coeffqpkwad500[1][0] / p ** coeffqpkwad500[1][1]), ] y_875 = [ math.log(-coeffqpkwad875[0][0] / p ** coeffqpkwad875[0][1]), math.log(-coeffqpkwad875[1][0] / p ** coeffqpkwad875[1][1]), ] A_500 = [ [1, -math.log(coeffqpkwad500[0][2] * p**2)], [1, -math.log(coeffqpkwad500[1][2] * p**2)], ] A_875 = [ [1, -math.log(coeffqpkwad875[0][2] * p**2)], [1, -math.log(coeffqpkwad875[1][2] * p**2)], ] # solve the set of equations labelled _500 eqs = normal_eqns.linear_ls(2) for i in range(2): eqs.add_equation( right_hand_side=y_500[i], design_matrix_row=flex.double(A_500[i]), weight=1, ) eqs.solve() coeffic_500 = eqs.solution() # solve the set of equations labelled _875 eqs = normal_eqns.linear_ls(2) for i in range(2): eqs.add_equation( right_hand_side=y_875[i], design_matrix_row=flex.double(A_875[i]), weight=1, ) eqs.solve() coeffic_875 = eqs.solution() fp_500_n = 1 - math.exp(coeffic_500[0]) / n ** coeffic_500[1] fp_875_n = 1 - math.exp(coeffic_875[0]) / n ** coeffic_875[1] elif p == 2: fp_500_n = 1 - math.exp(0.673292623522027) / n**0.691365864961895 fp_875_n = 1 - math.exp(0.446537815635445) / n**1.06690782995919 elif p == 1: fp_500_n = 1 - math.exp(0.262024211897096) / n**0.604756680630497 fp_875_n = 1 - math.exp(-0.351584646688712) / n**1.01646567502486 if alpha <= 0.875: fp_alpha_n = fp_500_n + (fp_875_n - fp_500_n) / 0.375 * (alpha - 0.5) if 0.875 < alpha and alpha <= 1: fp_alpha_n = fp_875_n + (1 - fp_875_n) / 0.125 * (alpha - 0.875) return 1 / fp_alpha_n