def search(self, rmax): z_model = zernike_model(self.nlm_array, self.data.q, rmax / self.fraction, self.nmax) self.nn_array.load_coefs(self.nn, self.this_coef) i_cal = z_model.calc_intensity(self.nn_array) scale = i_cal[0] i_cal = i_cal / (scale / self.int_scale) if (self.smear): new_i = smear_data(i_cal, self.data.q, self.delta_q) self.this_score = flex.sum_sq( (new_i - self.data.i[1:-2]) / self.weights[1:-2]) else: self.this_score = flex.sum_sq((i_cal - self.data.i) / self.weights) self.this_rmax = rmax return self.this_score
def check_i_obs_vs_backup(O, work_params): print "Current i_obs vs. backup:" for im in O.array: im.backup.reset_partialities(work_params, O.miller_indices) b_obs = im.backup.extract_i_obs_est(work_params, O.miller_indices) im.reset_partialities(work_params, O.miller_indices) i_obs = im.extract_i_obs_est(work_params, O.miller_indices) max_common_size = -1 max_cb_ci = None for s in work_params.lattice_symmetry.group(): i_obs_cb = i_obs.change_basis(str(s)) cb, ci = b_obs.common_sets(other=i_obs_cb) common_size = cb.indices().size() if (max_common_size < common_size): max_common_size = common_size max_cb_ci = cb, ci assert max_cb_ci is not None cb, ci = max_cb_ci from scitbx.array_family import flex num = flex.sum(cb.data()*ci.data()) den = flex.sum_sq(cb.data()) if (den == 0): scale = None else: scale = num / den print " ", b_obs.indices().size(), i_obs.indices().size(), \ cb.indices().size(), scale print
def build_cluster_pair_info(O, other, work_params, reindexing_assistant): from scitbx.array_family import flex scale_max = work_params.scale_estimation_scale_max assert scale_max > 0 scale_min = 1/scale_max miis_i, esti_i = O.miis_perms[0], O.esti_perms[0] result = [] for j_perm in xrange(len(reindexing_assistant.cb_ops)): miis_j, esti_j = other.miis_perms[j_perm], other.esti_perms[j_perm] i_seqs, j_seqs = miis_i.intersection_i_seqs(other=miis_j) if (i_seqs.size() < 2): return None x = esti_i.select(i_seqs) y = esti_j.select(j_seqs) if (((x != 0) | (y != 0)).count(True) < 2): return None num = flex.sum(x*y) den = flex.sum_sq(x) if (num > den * scale_min and num < den * scale_max): scale = num / den rms = flex.mean_sq(x*scale-y)**0.5 result.append(perm_rms_info(n=x.size(), scale=scale, rms=rms)) else: return None result = perm_rms_list(array=result) result.set_score() return result
def __init__(O, points, epsilon=1e-2): """\ Computation of Minimum-Volume Covering Ellipsoid using the Khachiyan Algorithm. Based on a Python implementation by Raj Rajashankar (ANL, Nov 2011), which in turn was based on a Matlab script by Nima Moshtagh (2009, http://stackoverflow.com/questions/1768197/bounding-ellipse/1768440#1768440). Caveats: - center and radii are correct, but rotation may permute axes - scales with the square of the number of points """ d = 3 # d is the dimension n = points.size() assert n > 0 # from scitbx.array_family import flex p = points.as_double() p.reshape(flex.grid(n, 3)) p = p.matrix_transpose() q = p.deep_copy() q.resize(flex.grid(4, n), 1) # u = flex.double(n, 1/n) umx = flex.double(flex.grid(n, n), 0) # err = epsilon + 1 while (err > epsilon): umx.matrix_diagonal_set_in_place(u) x_inv = q.matrix_multiply(umx).matrix_multiply_transpose( q).matrix_inversion() m = q.matrix_transpose_multiply( x_inv).matrix_multiply(q).matrix_diagonal() j = flex.max_index(m) maximum = m[j] ascent_step_size = (maximum-d-1)/((d+1)*(maximum-1)) new_u = (1 - ascent_step_size) * u new_u[j] += ascent_step_size err = flex.sum_sq(new_u - u)**0.5 u = new_u # center = p.matrix_multiply(u) umx.matrix_diagonal_set_in_place(u) t1 = p.matrix_multiply(umx).matrix_multiply_transpose(p) t2 = center.matrix_outer_product(center) a = (t1 - t2).matrix_inversion() / d # import scitbx.linalg.svd svd = scitbx.linalg.svd.real(a, accumulate_u=False, accumulate_v=True) size = 1.0/flex.sqrt(svd.sigma) from scitbx import matrix O.center = matrix.col(center) O.radii = matrix.col(size) O.rotation = matrix.sqr(svd.v)
def refinement_target(O, partiality_threshold): assert O.miller_image_map.map is not None from scitbx.array_family import flex result_num = 0 result_den = 0 for iiis in O.miller_image_map.map: estis = collect_estis(O.array, iiis, partiality_threshold) if (estis.size() < 2): continue i_obs_est = flex.mean(estis) result_num += flex.sum_sq(estis - i_obs_est) result_den += estis.size() return result_num / max(1, result_den)
def __init__(O, points, epsilon=1e-2): """\ Computation of Minimum-Volume Covering Ellipsoid using the Khachiyan Algorithm. Based on a Python implementation by Raj Rajashankar (ANL, Nov 2011), which in turn was based on a Matlab script by Nima Moshtagh (2009, http://stackoverflow.com/questions/1768197/bounding-ellipse/1768440#1768440). Caveats: - center and radii are correct, but rotation may permute axes - scales with the square of the number of points """ d = 3 # d is the dimension n = points.size() assert n > 0 # from scitbx.array_family import flex p = points.as_double() p.reshape(flex.grid(n, 3)) p = p.matrix_transpose() q = p.deep_copy() q.resize(flex.grid(4, n), 1) # u = flex.double(n, 1 / n) umx = flex.double(flex.grid(n, n), 0) # err = epsilon + 1 while (err > epsilon): umx.matrix_diagonal_set_in_place(u) x_inv = q.matrix_multiply(umx).matrix_multiply_transpose( q).matrix_inversion() m = q.matrix_transpose_multiply(x_inv).matrix_multiply( q).matrix_diagonal() j = flex.max_index(m) maximum = m[j] ascent_step_size = (maximum - d - 1) / ((d + 1) * (maximum - 1)) new_u = (1 - ascent_step_size) * u new_u[j] += ascent_step_size err = flex.sum_sq(new_u - u)**0.5 u = new_u # center = p.matrix_multiply(u) umx.matrix_diagonal_set_in_place(u) t1 = p.matrix_multiply(umx).matrix_multiply_transpose(p) t2 = center.matrix_outer_product(center) a = (t1 - t2).matrix_inversion() / d # import scitbx.linalg.svd svd = scitbx.linalg.svd.real(a, accumulate_u=False, accumulate_v=True) size = 1.0 / flex.sqrt(svd.sigma) from scitbx import matrix O.center = matrix.col(center) O.radii = matrix.col(size) O.rotation = matrix.sqr(svd.v)
def search(self, z_model, coefs, ntop, rmax): scores = flex.double() scales = flex.double() fancy_weight_obj = error_model.two_stage_basic(self.data.q, self.data.i, self.data.s, 0.02, 0.75) w = fancy_weight_obj.error_model_smear(rmax, 3) w = flex.sqrt(w) for coef in coefs: self.nn_array.load_coefs(self.nn, coef) i_cal = z_model.calc_intensity(self.nn_array) scale = i_cal[0] i_cal = i_cal / (scale / self.int_scale) scale = self.approx_scale(self.data.i, i_cal, w) if (self.smear): new_i = smear_data(i_cal, self.data.q, self.delta_q) score = flex.sum_sq( (new_i - scale * self.data.i[1:-2]) / (scale * w[1:-2])) #score = 1.0 - self.score_cc( new_i,self.data.i[1:-2], self.weights[1:-2] ) else: score = flex.sum_sq( (i_cal - self.data.i * scale) / (scale * w)) #score = 1.0 - self.score_cc( new_i,self.data.i,self.weights ) scores.append(score) scales.append(scale) top_hits = self.tops(scores, ntop) top_scores = flex.double() top_scales = flex.double() for tt in top_hits: top_scores.append(scores[tt]) top_scales.append(scales[tt]) return top_hits, top_scores, top_scales
def target(self, vector): self.counter += 1 result = 0 length = flex.sum_sq(vector) if (length > self.Rmax2): result = 1e30 else: new_coord = self.pdb.NMPerturb(self.modes, vector) self.she_engine.engine.update_coord(new_coord, self.new_indx) new_I = self.she_engine.engine.I() var = self.expt_s s, o = she.linear_fit(new_I[:5], self.expt_I[:5], var[:5]) result = flex.sum( flex.pow2((self.expt_I - (s * new_I + o)) / self.expt_s)) # result = flex.sum_sq( (self.expt_I-new_I) /self.expt_s ) #print self.pdb.r, result return result
def randomize_rotations(self): hypersphere_point = flex.double(4) self.rotations = [ None for i in xrange(len(self.species)) ] for i in xrange(len(self.species)): self.rotations[i] = [ None for j in xrange(self.species[i].n_copies) ] for j in xrange(self.species[i].n_copies): # pick random point on a 3-sphere for k in xrange(hypersphere_point.size()): hypersphere_point[k] = self.random.normalvariate(0.0,1.0) hypersphere_point = hypersphere_point /\ math.sqrt(flex.sum_sq(hypersphere_point)) a = hypersphere_point[0] b = hypersphere_point[1] c = hypersphere_point[2] d = hypersphere_point[3] # convert quaternion to rotation matrix self.rotations[i][j] =\ (a*a + b*b - c*c - d*d, 2*b*c - 2*a*d, 2*b*d + 2*a*c, 2*b*c + 2*a*d, a*a - b*b + c*c - d*d, 2*c*d - 2*a*b, 2*b*d - 2*a*c, 2*c*d + 2*a*b , a*a - b*b - c*c + d*d)
def search(self, z_model, coefs, ntop, rmax): scores = flex.double() scales = flex.double() w = 1.0 ## temporary use for coef in coefs: self.nlm_array.load_coefs(self.nlm, coef) calc_blq = z_model.get_all_blq(self.nlm_array) scale = calc_blq[0] calc_blq = calc_blq / scale scale = self.approx_scale(self.data.blq, calc_blq, w) score = flex.sum_sq( (calc_blq - self.data.blq * scale) / (scale * w)) scores.append(score) scales.append(scale) top_hits = self.tops(scores, ntop) top_scores = flex.double() top_scales = flex.double() for tt in top_hits: top_scores.append(scores[tt]) top_scales.append(scales[tt]) return top_hits, top_scores, top_scales
def target(self, x): self.calc_data=self.calc_Cnm_from_Inm(x) score = flex.sum_sq( self.calc_data-self.target_data) return score/self.scale
def functional(self, x=None, f_x=None): if (f_x is None): f_x = self.f(x=x) return 0.5*flex.sum_sq(f_x)
def write_html_summary(fname, atom_group_summaries): # Get template to be filled in template = GIANT_HTML_ENV.get_template('summary_page.html') # ===========================================================> # Construct the data object to populate the template output_data = {} output_data['header'] = 'Structure Summary' output_data['title'] = 'Structure Summary' # # ===========================================================> # # Progress Bars # output_data['progress_bar'] = [] # output_data['progress_bar'].append({'title':'Dataset Summary', 'data':[]}) # output_data['progress_bar'][0]['data'].append({'text':'Interesting', 'colour':'success', 'size':100.0*num_interesting/len_data}) # output_data['progress_bar'][0]['data'].append({'text':'Not Interesting', 'colour':'info', 'size':100.0*num_not_interesting/len_data}) # output_data['progress_bar'][0]['data'].append({'text':'Not Analysed', 'colour':'warning', 'size':100.0*num_not_analysed/len_data}) # output_data['progress_bar'][0]['data'].append({'text':'Rejected', 'colour':'danger', 'size':100.0*num_rejected/len_data}) # ===========================================================> # Tables output_data['table'] = {} output_data['table']['column_headings'] = [ 'Num Atoms', 'Min BFactor', 'Max BFactor', 'Mean BFactor', 'RMS BFactor', 'Min BFactor-Z', 'Max BFactor-Z', 'Mean BFactor-Z', 'RMS BFactor-Z', ] output_data['table']['rows'] = [] for ag_sum in atom_group_summaries: # Extract the atom group info as a dictionary ed_sum = ag_sum.edstats_scores bfs = ag_sum.b_factors bfs_stat = ag_sum.b_factors_statistics bfz = ag_sum.b_factors_z bfz_stat = ag_sum.b_factors_z_statistics # colour choices - 'success', 'muted', 'danger' # icon choices - 'ok', 'flag', 'remove' columns = [] # Num Atoms columns.append({'colour': 'default', 'message': round(bfs.size(), 1)}) # B-Factors columns.append({ 'colour': 'default', 'message': round(float(bfs_stat.min), 1) }) columns.append({ 'colour': 'default', 'message': round(float(bfs_stat.max), 1) }) columns.append({ 'colour': 'default', 'message': round(float(bfs_stat.mean), 1) }) columns.append({ 'colour': 'default', 'message': round(float(flex.sum_sq(bfs) / bfs.size())**0.5, 1) }) # B-Factors Z-Scores columns.append({ 'colour': 'default', 'message': round(float(bfz_stat.min), 2) }) columns.append({ 'colour': 'default', 'message': round(float(bfz_stat.max), 2) }) columns.append({ 'colour': 'default', 'message': round(float(bfz_stat.mean), 2) }) if (flex.sum_sq(bfz) / bfz.size())**0.5 < 2.5: columns.append({ 'colour': 'default', 'message': round(float(flex.sum_sq(bfz) / bfz.size())**0.5, 1) }) else: columns.append({ 'colour': 'danger', 'icon': 'remove', 'message': round((flex.sum_sq(bfz) / bfz.size())**0.5, 1) }) row_message = '' if (flex.sum_sq(bfz)/bfz.size())**0.5 < 2.5 else \ 'Check' row_colour = 'success' if row_message == '' else \ 'danger' output_data['table']['rows'].append({ 'heading': ag_sum.atom_group.id_str(), 'message': row_message, 'colour': row_colour, 'columns': columns }) with open(fname, 'w') as out_html: out_html.write(template.render(output_data))
def __init__(self, function, x0, mu0=None, tau=1.e-3, eps_1=1.e-16, eps_2=1.e-16, mu_min=1.e-300, k_max=1000): self.function = function x = x0 f_x = function.f(x=x) number_of_function_evaluations = 1 self.f_x0 = f_x fp = function.gradients(x=x, f_x=f_x) number_of_gradient_evaluations = 1 number_of_hessian_evaluations = 0 number_of_cholesky_decompositions = 0 mu = mu0 nu = 2 k = 0 while (k < k_max): if (flex.max(flex.abs(fp)) <= eps_1): break fdp = function.hessian(x=x) number_of_hessian_evaluations += 1 if (mu is None): mu = tau * flex.max(flex.abs(fdp)) if (mu == 0): mu = fp.norm()/max(x.norm(), floating_point_epsilon_double**0.5) fdp_plus_mu = fdp.deep_copy() if (mu > mu_min): fdp_plus_mu.matrix_diagonal_add_in_place(value=mu) u = fdp_plus_mu.matrix_symmetric_as_packed_u() gmw = scitbx.linalg.gill_murray_wright_cholesky_decomposition_in_place(u) number_of_cholesky_decompositions += 1 h_dn = gmw.solve(b=-fp) if (h_dn.norm() <= eps_2*(eps_2 + x.norm())): break x_new = x + h_dn f_x_new = function.f(x=x_new) number_of_function_evaluations += 1 fp_new = function.gradients(x=x_new, f_x=f_x_new) number_of_gradient_evaluations += 1 f = flex.sum_sq(f_x) fn = flex.sum_sq(f_x_new) df = f - fn accept = 0 if (df > 0): accept = 1 dl = 0.5 * h_dn.dot(h_dn * mu - fp) elif (fn <= f + abs(f)*(1+100*floating_point_epsilon_double)): df = (fp + fp_new).dot(fp - fp_new) if (df > 0): accept = 2 if (accept != 0): if (accept == 1 and dl > 0): if (mu > mu_min): mu *= max(1/3, 1 - (2*df/dl - 1)**3) else: mu = mu_min nu = 2 else: mu *= nu nu *= 2 x = x_new f_x = f_x_new fp = fp_new fdp = None k += 1 else: mu *= nu nu *= 2 self.x_star = x self.f_x_star = f_x self.number_of_iterations = k self.number_of_function_evaluations = number_of_function_evaluations self.number_of_gradient_evaluations = number_of_gradient_evaluations self.number_of_hessian_evaluations = number_of_hessian_evaluations self.number_of_cholesky_decompositions = number_of_cholesky_decompositions
def target(self, r): z_model = zernike_model(trivial_nlm_array, self.q_array, r, self.nmax) calc_i = z_model.calc_intensity(self.nn_array) calc_i = calc_i / calc_i[0] return flex.sum_sq(self.ref_int - calc_i)
def __init__(self, function, x0, mu0=None, tau=1.e-3, eps_1=1.e-16, eps_2=1.e-16, mu_min=1.e-300, k_max=1000): self.function = function x = x0 f_x = function.f(x=x) number_of_function_evaluations = 1 self.f_x0 = f_x fp = function.gradients(x=x, f_x=f_x) number_of_gradient_evaluations = 1 number_of_hessian_evaluations = 0 number_of_cholesky_decompositions = 0 mu = mu0 nu = 2 k = 0 while (k < k_max): if (flex.max(flex.abs(fp)) <= eps_1): break fdp = function.hessian(x=x) number_of_hessian_evaluations += 1 if (mu is None): mu = tau * flex.max(flex.abs(fdp)) if (mu == 0): mu = fp.norm() / max(x.norm(), floating_point_epsilon_double**0.5) fdp_plus_mu = fdp.deep_copy() if (mu > mu_min): fdp_plus_mu.matrix_diagonal_add_in_place(value=mu) u = fdp_plus_mu.matrix_symmetric_as_packed_u() gmw = scitbx.linalg.gill_murray_wright_cholesky_decomposition_in_place( u) number_of_cholesky_decompositions += 1 h_dn = gmw.solve(b=-fp) if (h_dn.norm() <= eps_2 * (eps_2 + x.norm())): break x_new = x + h_dn f_x_new = function.f(x=x_new) number_of_function_evaluations += 1 fp_new = function.gradients(x=x_new, f_x=f_x_new) number_of_gradient_evaluations += 1 f = flex.sum_sq(f_x) fn = flex.sum_sq(f_x_new) df = f - fn accept = 0 if (df > 0): accept = 1 dl = 0.5 * h_dn.dot(h_dn * mu - fp) elif (fn <= f + abs(f) * (1 + 100 * floating_point_epsilon_double)): df = (fp + fp_new).dot(fp - fp_new) if (df > 0): accept = 2 if (accept != 0): if (accept == 1 and dl > 0): if (mu > mu_min): mu *= max(1 / 3, 1 - (2 * df / dl - 1)**3) else: mu = mu_min nu = 2 else: mu *= nu nu *= 2 x = x_new f_x = f_x_new fp = fp_new fdp = None k += 1 else: mu *= nu nu *= 2 self.x_star = x self.f_x_star = f_x self.number_of_iterations = k self.number_of_function_evaluations = number_of_function_evaluations self.number_of_gradient_evaluations = number_of_gradient_evaluations self.number_of_hessian_evaluations = number_of_hessian_evaluations self.number_of_cholesky_decompositions = number_of_cholesky_decompositions
def test_significance_of_group_of_z_values(vals): """Test the significance of a group of z-values - assumes sum-squared is distributed as a chi-squared""" sum_sq = flex.sum_sq(flex.double(vals)) n = len(vals) return chi_squared_test(sum_sq, n)