def plot_positions(values, positions, file_name, cmap=pyplot.cm.Reds, vmin=None, vmax=None, invalid='white'): values = values.as_double() assert positions.size() >= values.size() positions = positions[:values.size()] if vmin is None: vmin = flex.min(values) if vmax is None: vmax = flex.max(values) x, y = positions.parts() dx = flex.abs(x[1:] - x[:-1]) dy = flex.abs(y[1:] - y[:-1]) dx = dx.select(dx > 0) dy = dy.select(dy > 0) scale = 1/flex.min(dx) #print scale x = (x * scale).iround() y = (y * scale).iround() from libtbx.math_utils import iceil z = flex.double(flex.grid(iceil(flex.max(y))+1, iceil(flex.max(x))+1), -2) #print z.all() for x_, y_, z_ in zip(x, y, values): z[y_, x_] = z_ plot_grid(z.as_1d(), z.all(), file_name, cmap=cmap, vmin=vmin, vmax=vmax, invalid=invalid) return
def test_grid_step(n_sites = 50, volume_per_atom = 50, d_min = 2.0): grid_step = (0.2,0.4,0.6,0.7,0.9,1.0) for step in grid_step: symmetry = crystal.symmetry(space_group_symbol="P1") structure = random_structure.xray_structure(space_group_info = symmetry.space_group_info(), elements=["C"]*n_sites, volume_per_atom=volume_per_atom, random_u_iso=False) fc = structure.structure_factors(d_min = d_min, anomalous_flag=False, algorithm="fft").f_calc() manager = max_like_non_uniform.ordered_solvent_distribution( structure = structure, fo = fc, grid_step = step) f_water_dist = manager.fcalc_from_distribution() ### check phase compatibility with the symmetry: centrics = f_water_dist.select_centric() if(centrics.indices().size() > 0): ideal = centrics.phase_transfer(centrics) assert flex.max(flex.abs(ideal.data() - centrics.data())) < 1.e-6 ### #print "max = ", flex.max( flex.abs( f_water_dist.data() ) ) #print "min = ", flex.min( flex.abs( f_water_dist.data() ) ) #print "ave = ", flex.mean( flex.abs( f_water_dist.data() ) ) assert flex.max( flex.abs( f_water_dist.data() ) ) < 1.0
def __init__(self, intensities, dose, n_bins=8, range_min=None, range_max=None, range_width=1): self.intensities = intensities self.dose = dose self.n_bins = n_bins self.range_min = range_min self.range_max = range_max self.range_width = range_width assert self.range_width > 0 if self.range_min is None: self.range_min = flex.min(self.dose) - self.range_width if self.range_max is None: self.range_max = flex.max(self.dose) self.n_steps = 2 + int((self.range_max - self.range_min) - self.range_width) sel = (self.dose.as_double() <= self.range_max) & (self.dose.as_double() >= self.range_min) self.dose = self.dose.select(sel) self.intensities = self.intensities.select(sel) self.d_star_sq = self.intensities.d_star_sq().data() self.binner = self.intensities.setup_binner_d_star_sq_step( d_star_sq_step=(flex.max(self.d_star_sq)-flex.min(self.d_star_sq)+1e-8)/self.n_bins) self.observations = unmerged_observations(self.intensities) self._calc_completeness_vs_dose() self._calc_rcp_scp() self._calc_rd()
def linear_regression_test(d_analytical, d_numerical, test_hard=True, slope_tolerance=1.e-3, correlation_min=0.999, verbose=0): if (type(d_analytical) != type(flex.double())): d_analytical = flex_tuple_as_flex_double(d_analytical) if (type(d_numerical) != type(flex.double())): d_numerical = flex_tuple_as_flex_double(d_numerical) if (0 or verbose): print "analytical:", tuple(d_analytical) print "numerical: ", tuple(d_numerical) if ( flex.max(flex.abs(d_analytical)) == 0 and flex.max(flex.abs(d_numerical)) == 0): return regr = flex.linear_regression(d_analytical, d_numerical) corr = flex.linear_correlation(d_analytical, d_numerical).coefficient() assert regr.is_well_defined() if (abs(regr.slope() - 1) > slope_tolerance or corr < correlation_min): print "Error: finite difference mismatch:" print "slope:", regr.slope() print "correlation:", corr if (0 or verbose): for a, n in zip(d_analytical, d_numerical): print a, n assert not test_hard
def miller_array_export_as_shelx_hklf( miller_array, file_object=None, normalise_if_format_overflow=False): """\ If the maximum data value does not fit into the f8.2/f8.0 format: normalise_if_format_overflow=False: RuntimeError is thrown normalise_if_format_overflow=True: data is normalised to the largest number to fit f8.2/f8.0 format """ assert miller_array.is_real_array() if (file_object is None): file_object = sys.stdout def raise_f8_overflow(v): raise RuntimeError("SHELX HKL file F8.2/F8.0 format overflow: %.8g" % v) data = miller_array.data() sigmas = miller_array.sigmas() assert data is not None min_val = flex.min(data) max_val = flex.max(data) if (sigmas is not None): min_val = min(min_val, flex.min(sigmas)) max_val = max(max_val, flex.max(sigmas)) min_sc = 1 max_sc = 1 if (min_val < 0): s = "%8.0f" % min_val if (len(s) > 8): if (not normalise_if_format_overflow): raise_f8_overflow(min_val) min_sc = -9999999. / min_val if (max_val > 0): s = "%8.0f" % max_val if (len(s) > 8): if (not normalise_if_format_overflow): raise_f8_overflow(max_val) max_sc = 99999999. / max_val scale = min(min_sc, max_sc) sigmas = miller_array.sigmas() s = 0.01 for i,h in enumerate(miller_array.indices()): if (sigmas is not None): s = sigmas[i] def fmt_3i4(h): result = "%4d%4d%4d" % h if (len(result) != 12): raise RuntimeError( "SHELXL HKL file 3I4 format overflow: %s" % result) return result def fmt_f8(v): result = "%8.2f" % v if (len(result) != 8): result = "%8.1f" % v if (len(result) != 8): result = "%8.0f" % v assert len(result) == 8 return result line = fmt_3i4(h) + fmt_f8(data[i]*scale) + fmt_f8(s*scale) print >> file_object, line print >> file_object, " 0 0 0 0.00 0.00"
def verify_miller_arrays(a1, a2, eps=1.0e-5): v = a2.adopt_set(a1) if a1.is_bool_array(): if a2.is_integer_array(): assert flex.max(flex.abs(a1.data().as_int() - v.data())) == 0 else: assert flex.max(flex.abs(a1.data().as_double() - v.data())) < eps elif a1.is_hendrickson_lattman_array(): for i in xrange(4): assert flex.max(flex.abs(a1.data().slice(i) - v.data().slice(i))) < eps else: assert flex.max(flex.abs(a1.data() - v.data())) < eps if v.sigmas() is not None: assert flex.max(flex.abs(a1.sigmas() - v.sigmas())) < eps
def minimize_kb(self, use_curvatures_options, set_use_scale_options=[True, False], n_cycles=5): #print "minimize_kb, r:", self.kbu.r_factor() for use_curvatures in use_curvatures_options*n_cycles: start_r = self.kbu.r_factor() save_k_sols = self.kbu.k_sols() save_b_sols = self.kbu.b_sols() #self.set_use_scale(value = random.choice(set_use_scale_options)) self.set_use_scale(value = True) m = self.minimize_kb_once(use_curvatures=use_curvatures) r = self.kbu.r_factor() if(r>start_r and r>1.e-2 and (flex.min(self.kbu.k_sols())<0 or flex.max(self.kbu.k_sols())>1 or flex.min(self.kbu.b_sols())<0 or flex.max(self.kbu.k_sols())>100.)): self.kbu.update(k_sols = save_k_sols, b_sols = save_b_sols)
def exercise_xray_structure(use_u_aniso, verbose=0): structure = random_structure.xray_structure( space_group_info=sgtbx.space_group_info("P 31"), elements=["N", "C", "C", "O", "Si"] * 2, volume_per_atom=500, min_distance=2.0, general_positions_only=False, random_u_iso=True, use_u_aniso=use_u_aniso, ) f_abs = abs(structure.structure_factors(anomalous_flag=False, d_min=2, algorithm="direct").f_calc()) for resname in (None, "res"): for fractional_coordinates in (False, True): pdb_file = structure.as_pdb_file( remark="Title", remarks=["Any", "Thing"], fractional_coordinates=fractional_coordinates, resname=resname ) if 0 or verbose: sys.stdout.write(pdb_file) structure_read = iotbx.pdb.input( source_info=None, lines=flex.std_string(pdb_file.splitlines()) ).xray_structure_simple(fractional_coordinates=fractional_coordinates, use_scale_matrix_if_available=False) f_read = abs( f_abs.structure_factors_from_scatterers(xray_structure=structure_read, algorithm="direct").f_calc() ) regression = flex.linear_regression(f_abs.data(), f_read.data()) assert regression.is_well_defined() if 0 or verbose: regression.show_summary() assert approx_equal(regression.slope(), 1, eps=1.0e-2) assert approx_equal(regression.y_intercept(), 0, eps=flex.max(f_abs.data()) * 0.01)
def show_literature_fits(label, n_terms, null_fit, n_points, e_other=None): for lib in [xray_scattering.wk1995, xray_scattering.it1992, xray_scattering.two_gaussian_agarwal_isaacs, xray_scattering.two_gaussian_agarwal_1978, xray_scattering.one_gaussian_agarwal_1978]: if (lib == xray_scattering.wk1995): try: lib_gaussian = xray_scattering.wk1995(label, True).fetch() lib_source = "WK1995" except Exception: lib_gaussian = None elif (lib == xray_scattering.it1992): try: lib_gaussian = xray_scattering.it1992(label, True).fetch() lib_source = "IT1992" except Exception: lib_gaussian = None elif (lib.table.has_key(label)): lib_gaussian = lib.table[label] lib_source = lib.source_short else: lib_gaussian = None if (lib_gaussian is not None): gaussian_fit = scitbx.math.gaussian.fit( null_fit.table_x()[:n_points], null_fit.table_y()[:n_points], null_fit.table_sigmas()[:n_points], lib_gaussian) e = flex.max(gaussian_fit.significant_relative_errors()) show_fit_summary(lib_source, label, gaussian_fit, e, e_other, lib_gaussian.n_terms())
def tst_hes_ls_f_wt(self,h=0.0000001): hes_anal = self.ls_f_wt.hessian_as_packed_u() hes_anal=hes_anal.matrix_packed_u_as_symmetric() grads = self.ls_f_wt.get_gradient() self.ls_f_wt.set_p_scale(self.p_scale+h) tmp = self.ls_f_wt.get_gradient() tmp = list( (grads-tmp)/-h ) tmp_hess=[] tmp_hess.append( tmp ) self.ls_f_wt.set_p_scale(self.p_scale) for ii in range(6): u_tmp=list(flex.double(self.u).deep_copy()) u_tmp[ii]+=h self.ls_f_wt.set_u_rwgk(u_tmp) tmp = self.ls_f_wt.get_gradient() tmp = (grads - tmp)/-h tmp_hess.append( list(tmp) ) self.ls_f_wt.set_u_rwgk(self.u) f = max(1, flex.max(flex.abs(hes_anal))) count=0 for ii in range(7): for jj in range(7): assert approx_equal(tmp_hess[ii][jj]/f, hes_anal[count]/f) count+=1
def gradients(self, xray_structure, force_update_mask=False): factor = 1.0 sites_cart = xray_structure.sites_cart() if(self.fmodel is not None): max_shift = flex.max(flex.sqrt((self.sites_cart - sites_cart).dot())) if(max_shift > self.update_gradient_threshold): self.fmodel.update_xray_structure( xray_structure = xray_structure, update_f_calc = True, update_f_mask = False) self.gx = flex.vec3_double(self.x_target_functor(compute_gradients=True).\ gradients_wrt_atomic_parameters(site=True).packed()) self.sites_cart = sites_cart if(self.restraints_manager is not None): c = self.restraints_manager.energies_sites(sites_cart = sites_cart, compute_gradients=True) self.gc = c.gradients factor *= self.wc if(c.normalization_factor is not None): factor *= c.normalization_factor result = None if(self.wx is not None): result = self.wx * self.gx if(self.wc is not None): gcw = self.wc * self.gc if(result is None): result = gcw else: result = result + gcw if(factor != 1.0): result *= 1.0 / factor #print "norms:", self.gc.norm(), self.gx.norm(), result.norm() return result
def plot_overall_completeness(completeness): completeness_range = xrange(-1, flex.max(completeness) + 1) completeness_counts = [completeness.count(n) for n in completeness_range] from matplotlib import pyplot as plt plt.plot(completeness_range, completeness_counts, "r+") plt.show()
def __init__(self, f_obs, ncs_pairs, reflections_per_bin): adopt_init_args(self, locals()) # Create bins f_obs.setup_binner(reflections_per_bin = reflections_per_bin) self.binner = f_obs.binner() n_bins = self.binner.n_bins_used() self.n_bins = n_bins self.SigmaN = None self.update_SigmaN() # self.rbin = flex.int(f_obs.data().size(), -1) for i_bin in self.binner.range_used(): for i_seq in self.binner.array_indices(i_bin): self.rbin[i_seq] = i_bin-1 # i_bin starts with 1, not 0 ! assert flex.min(self.rbin)==0 assert flex.max(self.rbin)==n_bins-1 # Extract symmetry matrices self.sym_matrices = [] for m_as_string in f_obs.space_group().smx(): o = sgtbx.rt_mx(symbol=str(m_as_string), t_den=f_obs.space_group().t_den()) m_as_double = o.r().as_double() self.sym_matrices.append(m_as_double) self.gradient_evaluator = None self.target_and_grads = ext.tncs_eps_factor_refinery( tncs_pairs = self.ncs_pairs, f_obs = self.f_obs.data(), sigma_f_obs = self.f_obs.sigmas(), rbin = self.rbin, SigmaN = self.SigmaN, space_group = self.f_obs.space_group(), miller_indices = self.f_obs.indices(), fractionalization_matrix = self.f_obs.unit_cell().fractionalization_matrix(), sym_matrices = self.sym_matrices) self.update()
def target_and_gradients(self, xray_structure, to_compute_weight=False): if(to_compute_weight): xrs = xray_structure.deep_copy_scatterers() # This may be useful to explore: #xrs.shake_adp_if_all_equal(b_iso_tolerance = 1.e-3) #xrs.shake_adp(spread=10, keep_anisotropic= False) else: xrs = xray_structure if(self.refine_adp): params = xrs.extract_u_iso_or_u_equiv() if(self.refine_occ): params = xrs.scatterers().extract_occupancies() if(to_compute_weight): pmin = flex.min(params) pmax = flex.max(params) if(abs(pmin-pmax)/abs(pmin+pmax)*2*100<1.e-3): pmean = flex.mean(params) n_par = params.size() params = flex.double() for i in xrange(n_par): params.append(pmean + 0.1 * pmean * random.choice([-1,0,1])) return crystal.adp_iso_local_sphere_restraints_energies( pair_sym_table = self.pair_sym_table, orthogonalization_matrix = self.orthogonalization_matrix, sites_frac = self.sites_frac, u_isos = params, selection = self.selection, use_u_iso = self.selection, grad_u_iso = self.selection, sphere_radius = self.sphere_radius, distance_power = 2, average_power = 1, min_u_sum = 1.e-6, compute_gradients = True, collect = False)
def exercise_1(grid_step, radius, shell, a, b, buffer_layer, site_cart): xray_structure = xray_structure_of_one_atom(site_cart = site_cart, buffer_layer = buffer_layer, a = a, b = b) sampled_density = mmtbx.real_space.sampled_model_density( xray_structure = xray_structure, grid_step = grid_step) site_frac = xray_structure.unit_cell().fractionalization_matrix()*site_cart assert approx_equal(flex.max(sampled_density.data()), sampled_density.data().value_at_closest_grid_point(site_frac[0])) around_atom_obj = mmtbx.real_space.around_atom( unit_cell = xray_structure.unit_cell(), map_data = sampled_density.data(), radius = radius, shell = shell, site_frac = list(xray_structure.sites_frac())[0]) data_exact = around_atom_obj.data() dist_exact = around_atom_obj.distances() approx_obj = maptbx.one_gaussian_peak_approximation( data_at_grid_points = data_exact, distances = dist_exact, use_weights = False, optimize_cutoff_radius = True) assert approx_equal(approx_obj.a_reciprocal_space(), 6.0, 1.e-3) assert approx_equal(approx_obj.b_reciprocal_space(), 3.0, 1.e-3) assert approx_obj.gof() < 0.3 assert approx_obj.cutoff_radius() < radius
def run_fast_nv1995(f_obs, f_calc_fixed, f_calc_p1, symmetry_flags, gridding, grid_tags, verbose): if (f_calc_fixed is None): f_part = flex.complex_double() else: f_part = f_calc_fixed.data() assert f_obs.anomalous_flag() == f_calc_p1.anomalous_flag() fast_nv1995 = translation_search.fast_nv1995( gridding=gridding, space_group=f_obs.space_group(), anomalous_flag=f_obs.anomalous_flag(), miller_indices_f_obs=f_obs.indices(), f_obs=f_obs.data(), f_part=f_part, miller_indices_p1_f_calc=f_calc_p1.indices(), p1_f_calc=f_calc_p1.data()) assert fast_nv1995.target_map().all() == gridding map_stats = maptbx.statistics(fast_nv1995.target_map()) if (0 or verbose): map_stats.show_summary() grid_tags.build(f_obs.space_group_info().type(), symmetry_flags) assert grid_tags.n_grid_misses() == 0 assert grid_tags.verify(fast_nv1995.target_map()) peak_list = maptbx.peak_list( data=fast_nv1995.target_map(), tags=grid_tags.tag_array(), peak_search_level=1, max_peaks=10, interpolate=True) if (0 or verbose): print "gridding:", gridding for i,site in enumerate(peak_list.sites()): print "(%.4f,%.4f,%.4f)" % site, "%.6g" % peak_list.heights()[i] assert approx_equal(map_stats.max(), flex.max(peak_list.grid_heights())) return peak_list
def quick_test(file_name): from libtbx.utils import user_plus_sys_time t = user_plus_sys_time() s = reader(file_name) print "Time read:", t.delta() s.show_summary() print tuple(s.original_indices[:3]) print tuple(s.unique_indices[:3]) print tuple(s.batch_numbers[:3]) print tuple(s.centric_tags[:3]) print tuple(s.spindle_flags[:3]) print tuple(s.asymmetric_unit_indices[:3]) print tuple(s.i_obs[:3]) print tuple(s.sigmas[:3]) print tuple(s.original_indices[-3:]) print tuple(s.unique_indices[-3:]) print tuple(s.batch_numbers[-3:]) print tuple(s.centric_tags[-3:]) print tuple(s.spindle_flags[-3:]) print tuple(s.asymmetric_unit_indices[-3:]) print tuple(s.i_obs[-3:]) print tuple(s.sigmas[-3:]) m = s.as_miller_array(merge_equivalents=False).merge_equivalents() print "min redundancies:", flex.min(m.redundancies().data()) print "max redundancies:", flex.max(m.redundancies().data()) print "mean redundancies:", flex.mean(m.redundancies().data().as_double()) s.as_miller_arrays()[0].show_summary() print
def show_plot(widegrid,excursi): excursi.reshape(flex.grid(widegrid, widegrid)) plot_max = flex.max(excursi) idx_max = flex.max_index(excursi) def igrid(x): return x - (widegrid//2) idxs = [igrid(i)*plot_px_sz for i in xrange(widegrid)] from matplotlib import pyplot as plt plt.figure() CS = plt.contour([igrid(i)*plot_px_sz for i in xrange(widegrid)], [igrid(i)*plot_px_sz for i in xrange(widegrid)], excursi.as_numpy_array()) plt.clabel(CS, inline=1, fontsize=10, fmt="%6.3f") plt.title("Wide scope search for detector origin offset") plt.scatter([0.0],[0.0],color='g',marker='o') plt.scatter([new_offset[0]] , [new_offset[1]],color='r',marker='*') plt.scatter([idxs[idx_max%widegrid]] , [idxs[idx_max//widegrid]],color='k',marker='s') plt.axes().set_aspect("equal") plt.xlabel("offset (mm) along beamr1 vector") plt.ylabel("offset (mm) along beamr2 vector") plt.show() #changing value trial_origin_offset = (idxs[idx_max%widegrid])*beamr1 + (idxs[idx_max//widegrid])*beamr2 return trial_origin_offset
def main(): parser = OptionParser(usage="usage: python %prog [options] file_name ...") parser.add_option("-c", "--cutoff", type="float", default=6.05, metavar="FLOAT", help="maximum sin(theta)/lambda") (options, args) = parser.parse_args() if len(args) < 1: parser.print_help() return cutoff = options.cutoff for file_name in args: tab = read_table(file_name) if tab.element == "Es": continue wk = xray_scattering.wk1995(tab.element, True).fetch() sel = tab.x <= cutoff tab_x = tab.x.select(sel) tab_y = tab.y.select(sel) sigmas = flex.double(tab_x.size(), 0.0005) wky = wk.at_x(tab_x) errors_abs = flex.abs(wky - tab_y) fit = scitbx.math.gaussian.fit(tab_x, tab_y, sigmas, wk) errors_rel = fit.significant_relative_errors(1.0e-6) print tab.element, tab.atomic_number, print "max error < %.1fA-1 abs, rel: %7.4f %7.4f" % (cutoff, flex.max(errors_abs), flex.max(errors_rel)) for x, y, f, ea, er in zip(tab_x, tab_y, wky, errors_abs, errors_rel): print "%7.4f %7.4f %7.4f %7.4f %7.4f" % (x, y, f, ea, er) print
def resolution_rmerge(self, limit = None, log = None): '''Compute a resolution limit where either rmerge = 1.0 (limit if set) or the full extent of the data. N.B. this fit is only meaningful for positive values.''' if limit is None: limit = self._params.rmerge rmerge_s = flex.double( [b.r_merge for b in self._merging_statistics.bins]).reversed() s_s = flex.double( [1/b.d_min**2 for b in self._merging_statistics.bins]).reversed() sel = rmerge_s > 0 rmerge_s = rmerge_s.select(sel) s_s = s_s.select(sel) if limit == 0.0: return 1.0 / math.sqrt(flex.max(s_s)) if limit > flex.max(rmerge_s): return 1.0 / math.sqrt(flex.max(s_s)) rmerge_f = log_inv_fit(s_s, rmerge_s, 6) if log: fout = open(log, 'w') for j, s in enumerate(s_s): d = 1.0 / math.sqrt(s) o = rmerge_s[j] m = rmerge_f[j] fout.write('%f %f %f %f\n' % (s, d, o, m)) fout.close() try: r_rmerge = 1.0 / math.sqrt(interpolate_value(s_s, rmerge_f, limit)) except: r_rmerge = 1.0 / math.sqrt(flex.max(s_s)) if self._params.plot: plot = resolution_plot(ylabel='Rmerge') plot.plot(s_s, rmerge_f, label='fit') plot.plot(s_s, rmerge_s, label='Rmerge') plot.plot_resolution_limit(r_rmerge) plot.savefig('rmerge.png') return r_rmerge
def show_xray_structure_statistics(xray_structure, atom_selections, hd_sel = None): result = group_args( all = None, macromolecule = None, sidechain = None, solvent = None, ligand = None, backbone = None) if(hd_sel is not None): xray_structure = xray_structure.select(~hd_sel) for key in atom_selections.__dict__.keys(): value = atom_selections.__dict__[key] if(value.count(True) > 0): if(hd_sel is not None): value = value.select(~hd_sel) xrs = xray_structure.select(value) atom_counts = xrs.scattering_types_counts_and_occupancy_sums() atom_counts_strs = [] for ac in atom_counts: atom_counts_strs.append("%s:%s:%s"%(ac.scattering_type,str(ac.count), str("%10.2f"%ac.occupancy_sum).strip())) atom_counts_str = " ".join(atom_counts_strs) b_isos = xrs.extract_u_iso_or_u_equiv() n_aniso = xrs.use_u_aniso().count(True) n_not_positive_definite = xrs.is_positive_definite_u().count(False) b_mean = format_value("%-6.1f",adptbx.u_as_b(flex.mean(b_isos))) b_min = format_value("%-6.1f",adptbx.u_as_b(flex.min(b_isos))) b_max = format_value("%-6.1f",adptbx.u_as_b(flex.max(b_isos))) n_atoms = format_value("%-8d",xrs.scatterers().size()).strip() n_npd = format_value("%-8s",n_not_positive_definite).strip() occ = xrs.scatterers().extract_occupancies() o_mean = format_value("%-6.2f",flex.mean(occ)).strip() o_min = format_value("%-6.2f",flex.min(occ)).strip() o_max = format_value("%-6.2f",flex.max(occ)).strip() tmp_result = group_args( n_atoms = n_atoms, atom_counts_str = atom_counts_str, b_min = b_min, b_max = b_max, b_mean = b_mean, o_min = o_min, o_max = o_max, o_mean = o_mean, n_aniso = n_aniso, n_npd = n_npd) setattr(result,key,tmp_result) return result
def get_pseudo_curvs(): ag_max = flex.max(flex.abs(inp_info.grads)) assert ag_max != 0 dests = (-inp_info.grads/ag_max) * (limits/2) assert flex.abs(dests).all_le(limits/2*(1+1e-6)) assert (dests > 0).all_eq(inp_info.grads < 0) O.pseudo_curvs_i_info = inp_i_info return dests
def __init__(self, unmerged_intensities, batches): self.unmerged_intensities = unmerged_intensities self.batches = batches self.minb = flex.min(self.batches.data()) self.maxb = flex.max(self.batches.data()) n_scale_factors = self.maxb-self.minb + 1 self.x = flex.double(n_scale_factors, 1) scitbx.lbfgs.run(target_evaluator=self)
def exercise_3(grid_step, radius, shell, a, b, d_min, site_cart, buffer_layer): xray_structure = xray_structure_of_one_atom(site_cart = site_cart, buffer_layer = buffer_layer, a = a, b = b) miller_set = miller.build_set( crystal_symmetry = xray_structure.crystal_symmetry(), anomalous_flag = False, d_min = d_min, d_max = None) f_calc = miller_set.structure_factors_from_scatterers( xray_structure = xray_structure, algorithm = "direct", cos_sin_table = False, exp_table_one_over_step_size = False).f_calc() fft_map = f_calc.fft_map(grid_step = grid_step, symmetry_flags = None) fft_map = fft_map.apply_volume_scaling() site_frac = xray_structure.sites_frac() r = abs(abs(flex.max(fft_map.real_map_unpadded()))-\ abs(fft_map.real_map_unpadded().value_at_closest_grid_point(site_frac[0])))/\ abs(flex.max(fft_map.real_map_unpadded())) * 100.0 assert approx_equal(r, 0.0) around_atom_obj_ = mmtbx.real_space.around_atom( unit_cell = xray_structure.unit_cell(), map_data = fft_map.real_map_unpadded(), radius = radius, shell = shell, site_frac = list(xray_structure.sites_frac())[0]) data = around_atom_obj_.data() dist = around_atom_obj_.distances() approx_obj_ = maptbx.one_gaussian_peak_approximation( data_at_grid_points = data, distances = dist, use_weights = False, optimize_cutoff_radius = True) assert approx_equal(approx_obj_.a_reciprocal_space(), 6.0, 0.1) assert approx_equal(approx_obj_.b_reciprocal_space(), 3.0, 0.01) assert approx_obj_.gof() < 0.6 assert approx_obj_.cutoff_radius() < radius
def test_4(): symmetry = crystal.symmetry(unit_cell = (15.67, 25.37, 35.68, 90, 90, 90), space_group_symbol = "P 21 21 21") structure = xray.structure(crystal_symmetry = symmetry) ma = structure.structure_factors(d_min = 1.5, anomalous_flag = False).f_calc() mi = ma.indices() # ================================================================= TEST-1 alpha = flex.double(mi.size()) beta = flex.double(mi.size()) d_obs = flex.double(mi.size()) d_calc = flex.double(mi.size()) # define test set reflections flags=flex.int(beta.size(), 0) k=0 for i in xrange(flags.size()): k=k+1 if (k !=10): flags[i]=0 else: k=0 flags[i]=1 for i in range(1,mi.size()+1): d_obs [i-1] = i*1.5 d_calc[i-1] = i*1.0 beta [i-1] = i*500.0 alpha [i-1] = float(i) / float(i + 1) obj = max_lik.fom_and_phase_error( f_obs = d_obs, f_model = d_calc, alpha = alpha, beta = beta, epsilons = ma.epsilons().data().as_double(), centric_flags = ma.centric_flags().data()) per = obj.phase_error() fom = obj.fom() assert approx_equal(flex.max(per) , 89.9325000127 , 1.e-4) assert approx_equal(flex.min(per) , 5.37565067746e-05 , 1.e-4) assert approx_equal(flex.mean(per), 20.7942460698 , 1.e-4) assert approx_equal(flex.max(fom) , 0.999999402705 , 1.e-4) assert approx_equal(flex.min(fom) , 0.000749999859375 , 1.e-4) assert approx_equal(flex.mean(fom), 0.858269037582 , 1.e-4)
def reset_max_error(itvc_entry, fit): sel = international_tables_stols <= fit.stol + 1.e-6 gaussian_fit = scitbx.math.gaussian.fit( international_tables_stols.select(sel), itvc_entry.table_y.select(sel), itvc_entry.table_sigmas.select(sel), fit) fit.max_error = flex.max(gaussian_fit.significant_relative_errors())
def anisotropic_correction(cache_0, p_scale, u_star, b_add=None, must_be_greater_than=0.): ## Make sure that u_star is not rwgk scaled, i.e. like you get it from ## the ml_absolute_scale_aniso routine (!which is !!NOT!! scaled!) work_array = None try: work_array = cache_0.input.select( cache_0.input.data() > must_be_greater_than) except KeyboardInterrupt: raise except Exception: pass if work_array is None: work_array = cache_0.select( cache_0.data() > must_be_greater_than) change_back_to_intensity=False if work_array.is_xray_intensity_array(): work_array = work_array.f_sq_as_f() change_back_to_intensity=True assert not work_array.is_xray_intensity_array() if b_add is not None: u_star_add = adptbx.b_iso_as_u_star( work_array.unit_cell(), b_add ) u_star = u_star+u_star_add corrected_amplitudes = scaling.ml_normalise_aniso( work_array.indices(), work_array.data(), p_scale, work_array.unit_cell(), u_star ) if work_array.sigmas() is not None: corrected_sigmas = scaling.ml_normalise_aniso( work_array.indices(), work_array.sigmas(), p_scale, work_array.unit_cell(), u_star ) else: corrected_sigmas = None work_array = work_array.customized_copy( data = corrected_amplitudes, sigmas = corrected_sigmas ).set_observation_type(work_array) if change_back_to_intensity: # XXX check for floating-point overflows (which trigger the Boost trap # and crash the interpreter). The only known case is 2q8o:IOBS2,SIGIOBS2 # which is missing nearly all acentric hkls but it clearly points to a bug # in this routine when dealing with pathological data. f_max = flex.max(work_array.data()) if (not f_max < math.sqrt(sys.float_info.max)) : raise OverflowError("Amplitudes will exceed floating point limit if "+ "converted to intensities (max F = %e)." % f_max) work_array = work_array.f_as_f_sq() return work_array
def accelerations(self): self.stereochemistry_residuals = self.restraints_manager.energies_sites( sites_cart=self.structure.sites_cart(), compute_gradients=True ) # Harmonic restraints if self.er_data is not None: if self.er_data.er_harmonic_restraints_info is not None: harmonic_grads = self.restraints_manager.geometry.ta_harmonic_restraints( sites_cart=self.structure.sites_cart(), ta_harmonic_restraint_info=self.er_data.er_harmonic_restraints_info, weight=self.er_data.er_harmonic_restraints_weight, slack=self.er_data.er_harmonic_restraints_slack, ) assert self.stereochemistry_residuals.gradients.size() == harmonic_grads.size() self.stereochemistry_residuals.gradients += harmonic_grads result = self.stereochemistry_residuals.gradients d_max = None if self.xray_structure_last_updated is not None and self.shift_update > 0: array_of_distances_between_each_atom = flex.sqrt( self.structure.difference_vectors_cart(self.xray_structure_last_updated).dot() ) d_max = flex.max(array_of_distances_between_each_atom) if self.fmodel is not None: if d_max is not None: if d_max > self.shift_update: self.xray_structure_last_updated = self.structure.deep_copy_scatterers() self.xray_gradient = self.xray_grads() else: self.xray_gradient = self.xray_grads() result = ( self.xray_gradient * self.xray_target_weight + self.stereochemistry_residuals.gradients * self.chem_target_weight ) factor = 1.0 if self.chem_target_weight is not None: factor *= self.chem_target_weight if self.stereochemistry_residuals.normalization_factor is not None: factor *= self.stereochemistry_residuals.normalization_factor if factor != 1.0: result *= 1.0 / factor # Store RMS non-solvent atom gradients for Xray and Geo if self.er_data is not None: self.wc = self.chem_target_weight / factor self.wx = self.xray_target_weight / factor self.gg = self.stereochemistry_residuals.gradients * self.wc self.xg = self.xray_gradient * self.wx gg_pro = self.gg.select(~self.er_data.solvent_sel) xg_pro = self.xg.select(~self.er_data.solvent_sel) self.er_data.geo_grad_rms += (flex.mean_sq(gg_pro.as_double()) ** 0.5) / self.n_steps self.er_data.xray_grad_rms += (flex.mean_sq(xg_pro.as_double()) ** 0.5) / self.n_steps return result
def exercise_2(): symmetry = crystal.symmetry( unit_cell=(5.67, 10.37, 10.37, 90, 135.49, 90), space_group_symbol="C2") structure = xray.structure(crystal_symmetry=symmetry) atmrad = flex.double() xyzf = flex.vec3_double() for k in xrange(100): scatterer = xray.scatterer( site = ((1.+k*abs(math.sin(k)))/1000.0, (1.+k*abs(math.cos(k)))/1000.0, (1.+ k)/1000.0), scattering_type = "C") structure.add_scatterer(scatterer) atmrad.append(van_der_waals_radii.vdw.table[scatterer.element_symbol()]) xyzf.append(scatterer.site) miller_set = miller.build_set( crystal_symmetry=structure, d_min=1.0, anomalous_flag=False) step = 0.5 crystal_gridding = maptbx.crystal_gridding( unit_cell=structure.unit_cell(), step=step) nxyz = crystal_gridding.n_real() shrink_truncation_radius = 1.0 solvent_radius = 1.0 m1 = around_atoms( structure.unit_cell(), structure.space_group().order_z(), structure.sites_frac(), atmrad, nxyz, solvent_radius, shrink_truncation_radius) assert m1.solvent_radius == 1 assert m1.shrink_truncation_radius == 1 assert flex.max(m1.data) == 1 assert flex.min(m1.data) == 0 assert m1.data.size() == m1.data.count(1) + m1.data.count(0) m2 = mmtbx.masks.bulk_solvent( xray_structure=structure, gridding_n_real=nxyz, ignore_zero_occupancy_atoms = False, solvent_radius=solvent_radius, shrink_truncation_radius=shrink_truncation_radius) assert m2.data.all_eq(m1.data) m3 = mmtbx.masks.bulk_solvent( xray_structure=structure, grid_step=step, ignore_zero_occupancy_atoms = False, solvent_radius=solvent_radius, shrink_truncation_radius=shrink_truncation_radius) assert m3.data.all_eq(m1.data) f_mask2 = m2.structure_factors(miller_set=miller_set) f_mask3 = m3.structure_factors(miller_set=miller_set) assert approx_equal(f_mask2.data(), f_mask3.data()) assert approx_equal(flex.sum(flex.abs(f_mask3.data())), 1095.17999134)
def fit_side_chain(self, clusters): rotamer_iterator = \ mmtbx.refinement.real_space.fit_residue.get_rotamer_iterator( mon_lib_srv = self.mon_lib_srv, residue = self.residue) if(rotamer_iterator is None): return selection = flex.size_t(flatten(clusters[0].vector)) if(self.target_map is not None): start_target_value = self.get_target_value( sites_cart = self.residue.atoms().extract_xyz(), selection = selection) sites_cart_start = self.residue.atoms().extract_xyz() sites_cart_first_rotamer = list(rotamer_iterator)[0][1] self.residue.atoms().set_xyz(sites_cart_first_rotamer) axes = [] atr = [] for i, angle in enumerate(self.chi_angles[0]): cl = clusters[i] axes.append(flex.size_t(cl.axis)) atr.append(flex.size_t(cl.atoms_to_rotate)) if(self.target_map is not None): ro = ext.fit( target_value = start_target_value, axes = axes, rotatable_points_indices = atr, angles_array = self.chi_angles, density_map = self.target_map, all_points = self.residue.atoms().extract_xyz(), unit_cell = self.unit_cell, selection = selection, sin_table = self.sin_cos_table.sin_table, cos_table = self.sin_cos_table.cos_table, step = self.sin_cos_table.step, n = self.sin_cos_table.n) else: ro = ext.fit( sites_cart_start = sites_cart_start.deep_copy(), axes = axes, rotatable_points_indices = atr, angles_array = self.chi_angles, all_points = self.residue.atoms().extract_xyz(), sin_table = self.sin_cos_table.sin_table, cos_table = self.sin_cos_table.cos_table, step = self.sin_cos_table.step, n = self.sin_cos_table.n) sites_cart_result = ro.result() if(sites_cart_result.size()>0): dist = None if(self.accept_only_if_max_shift_is_smaller_than is not None): dist = flex.max(flex.sqrt((sites_cart_start - sites_cart_result).dot())) if(dist is None): self.residue.atoms().set_xyz(sites_cart_result) else: if(dist is not None and dist < self.accept_only_if_max_shift_is_smaller_than): self.residue.atoms().set_xyz(sites_cart_result) else: self.residue.atoms().set_xyz(sites_cart_start)
def test_2(): n_sites = 1000 d_min = 2.0 volume_per_atom = 50 fraction_missing = (0.0, ) scale = 5.0 # create dummy model space_group_info = sgtbx.space_group_info("P212121") structure = random_structure.xray_structure( space_group_info=space_group_info, elements=["N"] * (n_sites), volume_per_atom=volume_per_atom, random_u_iso=False) structure.scattering_type_registry(table="wk1995") f_calc = structure.structure_factors(d_min=d_min, anomalous_flag=False, algorithm="direct").f_calc() f_obs = abs(f_calc) for fm in fraction_missing: # partial model n_keep = int(round(structure.scatterers().size() * (1 - fm))) partial_structure = xray.structure(special_position_settings=structure) partial_structure.add_scatterers(structure.scatterers()[:n_keep]) # fcalc (partial model), fobs (fcalc full model) f_calc_partial = partial_structure.structure_factors( d_min=d_min, anomalous_flag=False, algorithm="direct").f_calc() f_calc = abs(f_calc_partial) # define test set reflections flags = flex.bool(f_calc_partial.indices().size(), False) k = 0 for i in xrange(f_calc_partial.indices().size()): k = k + 1 if (k != 10): flags[i] = False else: k = 0 flags[i] = True # *********************************************************TEST = 1 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=f_obs, f_calc=f_calc, free_reflections_per_bin=f_obs.data().size(), flags=flags, interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) alpha, beta = maxlik.alpha_beta_est_manager( f_obs=f_obs, f_calc=f_calc, free_reflections_per_bin=f_obs.data().size(), flags=flags, interpolation=True, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) # *********************************************************TEST = 2 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flags, interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flags, interpolation=True, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) # *********************************************************TEST = 3 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flex.bool(f_obs.data().size(), True), interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flex.bool(f_obs.data().size(), True), interpolation=True, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) # *********************************************************TEST = 4 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flex.bool(f_obs.data().size(), False), interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flex.bool(f_obs.data().size(), False), interpolation=True, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
def fit_side_chain(self, clusters): rotamer_iterator = \ mmtbx.refinement.real_space.fit_residue.get_rotamer_iterator( mon_lib_srv = self.mon_lib_srv, residue = self.residue) if(rotamer_iterator is None): return selection_clash = self.co.clash_eval_selection selection_rsr = self.co.rsr_eval_selection if(self.target_map is not None): start_target_value = self.get_target_value( sites_cart = self.residue.atoms().extract_xyz(), selection = selection_rsr) sites_cart_start = self.residue.atoms().extract_xyz() sites_cart_first_rotamer = list(rotamer_iterator)[0][1] # From this point on the coordinates in residue are to initial rotamer! self.residue.atoms().set_xyz(sites_cart_first_rotamer) axes = [] atr = [] for i, angle in enumerate(self.chi_angles[0]): cl = clusters[i] axes.append(flex.size_t(cl.axis)) atr.append(flex.size_t(cl.atoms_to_rotate)) # if(self.target_map is not None and self.xyzrad_bumpers is not None): # Get reference map values ref_map_vals = flex.double() for a in self.residue.atoms(): key = "%s_%s_%s"%( a.parent().parent().parent().id, a.parent().resname, a.name.strip()) ref_map_vals.append(self.cmv[key]) # Get radii radii = mmtbx.refinement.real_space.get_radii( residue = self.residue, vdw_radii = self.vdw_radii) # Exclude rotatable H from clash calculation tmp = flex.size_t() for i in selection_clash: if(self.rotatable_hd[self.residue.atoms()[i].i_seq]): continue tmp.append(i) selection_clash = tmp[:] # Ad hoc: S or SE have larger peaks! if(self.residue.resname in ["MET","MSE"]): scale=100 else: scale=3 moving = ext.moving( sites_cart = self.residue.atoms().extract_xyz(), sites_cart_start = sites_cart_start, radii = radii, weights = self.weights, bonded_pairs = self.pairs, ref_map_max = ref_map_vals * scale, ref_map_min = ref_map_vals / 10) # ro = ext.fit( fixed = self.xyzrad_bumpers, axes = axes, rotatable_points_indices = atr, angles_array = self.chi_angles, density_map = self.target_map, moving = moving, unit_cell = self.unit_cell, selection_clash = selection_clash, selection_rsr = selection_rsr, # select atoms to compute map target sin_table = self.sin_cos_table.sin_table, cos_table = self.sin_cos_table.cos_table, step = self.sin_cos_table.step, n = self.sin_cos_table.n) elif(self.target_map is not None and self.xyzrad_bumpers is None): ro = ext.fit( target_value = start_target_value, axes = axes, rotatable_points_indices = atr, angles_array = self.chi_angles, density_map = self.target_map, all_points = self.residue.atoms().extract_xyz(), unit_cell = self.unit_cell, selection = selection_rsr, sin_table = self.sin_cos_table.sin_table, cos_table = self.sin_cos_table.cos_table, step = self.sin_cos_table.step, n = self.sin_cos_table.n) else: ro = ext.fit( sites_cart_start = sites_cart_start.deep_copy(), axes = axes, rotatable_points_indices = atr, angles_array = self.chi_angles, all_points = self.residue.atoms().extract_xyz(), sin_table = self.sin_cos_table.sin_table, cos_table = self.sin_cos_table.cos_table, step = self.sin_cos_table.step, n = self.sin_cos_table.n) sites_cart_result = ro.result() if(sites_cart_result.size()>0): dist = None if(self.accept_only_if_max_shift_is_smaller_than is not None): dist = flex.max(flex.sqrt((sites_cart_start - sites_cart_result).dot())) if(dist is None): self.residue.atoms().set_xyz(sites_cart_result) else: if(dist is not None and dist < self.accept_only_if_max_shift_is_smaller_than): self.residue.atoms().set_xyz(sites_cart_result) else: self.residue.atoms().set_xyz(sites_cart_start) else: self.residue.atoms().set_xyz(sites_cart_start) if(self.m): self.m.add(residue = self.residue, state = "fitting") # # tune up if(self.target_map is not None): tune_up( target_map = self.target_map, residue = self.residue, mon_lib_srv = self.mon_lib_srv, rotamer_manager = self.rotamer_manager.rotamer_evaluator, unit_cell = self.unit_cell, monitor = self.m, torsion_search_start = -30, torsion_search_stop = 30, torsion_search_step = 1)
def _map_mmm_str(self): return " ".join([("%8.3f" % m).strip() for m in [ flex.min(self.map_data), flex.max(self.map_data), flex.mean(self.map_data) ]])
def exercise_3(): #test torsion restraints for use_reference in ['True', 'False', 'top_out', 'None']: pdb_inp = iotbx.pdb.input(lines=flex.std_string( pdb_str_2.splitlines()), source_info=None) model = manager(model_input=pdb_inp, log=null_out()) grm = model.get_restraints_manager().geometry xrs2 = model.get_xray_structure() awl2 = model.get_hierarchy().atoms_with_labels() pdb2 = model.get_hierarchy() pdb_inp3 = iotbx.pdb.input(source_info=None, lines=pdb_str_3) xrs3 = pdb_inp3.xray_structure_simple() ph3 = pdb_inp3.construct_hierarchy() ph3.atoms().reset_i_seq() awl3 = ph3.atoms_with_labels() sites_cart_reference = flex.vec3_double() selection = flex.size_t() min_selection = flex.size_t() reference_names = [ "N", "CA", "CB", "CG", "CD", "NE", "CZ", "NH1", "NH2" ] minimize_names = ["CG", "CD", "NE", "CZ", "NH1", "NH2"] for a2, a3 in zip(tuple(awl2), tuple(awl3)): assert a2.resname == a3.resname assert a2.name == a3.name assert a2.i_seq == a3.i_seq if (a2.resname == "ARG" and a2.name.strip() in reference_names): selection.append(a2.i_seq) sites_cart_reference.append(a3.xyz) if a2.name.strip() in minimize_names: min_selection.append(a2.i_seq) assert selection.size() == len(reference_names) selection_bool = flex.bool(xrs2.scatterers().size(), min_selection) if (use_reference == 'True'): grm.add_chi_torsion_restraints_in_place( pdb_hierarchy=pdb2, sites_cart=sites_cart_reference, selection=selection, sigma=2.5) elif (use_reference == 'top_out'): grm.add_chi_torsion_restraints_in_place( pdb_hierarchy=pdb2, sites_cart=sites_cart_reference, selection=selection, sigma=2.5, limit=180.0, top_out_potential=True) elif (use_reference == 'None'): grm.add_chi_torsion_restraints_in_place( pdb_hierarchy=pdb2, sites_cart=sites_cart_reference, selection=selection, sigma=2.5) grm.remove_chi_torsion_restraints_in_place(selection=selection) d1 = flex.mean( flex.sqrt((xrs2.sites_cart().select(min_selection) - xrs3.sites_cart().select(min_selection)).dot())) print "distance start (use_reference: %s): %6.4f" % ( str(use_reference), d1) assert d1 > 4.0 assert approx_equal( flex.max( flex.sqrt((xrs2.sites_cart().select(~selection_bool) - xrs3.sites_cart().select(~selection_bool)).dot())), 0) from cctbx import geometry_restraints import mmtbx.refinement.geometry_minimization import scitbx.lbfgs grf = geometry_restraints.flags.flags(default=True) grf.nonbonded = False sites_cart = xrs2.sites_cart() minimized = mmtbx.refinement.geometry_minimization.lbfgs( sites_cart=sites_cart, correct_special_position_tolerance=1.0, geometry_restraints_manager=grm, sites_cart_selection=flex.bool(sites_cart.size(), min_selection), geometry_restraints_flags=grf, lbfgs_termination_params=scitbx.lbfgs.termination_parameters( max_iterations=5000)) xrs2.set_sites_cart(sites_cart=sites_cart) d2 = flex.mean( flex.sqrt((xrs2.sites_cart().select(min_selection) - xrs3.sites_cart().select(min_selection)).dot())) print "distance final (use_reference: %s): %6.4f" % ( str(use_reference), d2) if (use_reference in ['True', 'top_out']): assert d2 < 0.02, d2 else: assert d2 > 4.0, d2 assert approx_equal( flex.max( flex.sqrt((xrs2.sites_cart().select(~selection_bool) - xrs3.sites_cart().select(~selection_bool)).dot())), 0) #test torsion manipulation grm.remove_chi_torsion_restraints_in_place() grm.remove_chi_torsion_restraints_in_place() sites_cart_reference = [] selections_reference = [] for model in pdb2.models(): for chain in model.chains(): for residue in chain.residues(): sites_cart_reference.append(residue.atoms().extract_xyz()) selections_reference.append(residue.atoms().extract_i_seq()) #one residue at a time (effectively chi angles only) for sites_cart, selection in zip(sites_cart_reference, selections_reference): grm.add_chi_torsion_restraints_in_place(pdb_hierarchy=pdb2, sites_cart=sites_cart, selection=selection) assert grm.get_n_chi_torsion_proixes() == 6 grm.remove_chi_torsion_restraints_in_place() #all sites at once, chi angles only sites_cart = xrs2.sites_cart() grm.add_chi_torsion_restraints_in_place(pdb_hierarchy=pdb2, sites_cart=sites_cart, selection=None, chi_angles_only=True) assert grm.get_n_chi_torsion_proixes() == 6 #all sites at once, all torsions grm.add_chi_torsion_restraints_in_place(pdb_hierarchy=pdb2, sites_cart=sites_cart, selection=None, chi_angles_only=False) # grm.get_chi_torsion_proxies().show_sorted( # by_value='residual', # sites_cart=sites_cart, # site_labels=[atom.id_str() for atom in pdb2.atoms()]) assert grm.get_n_chi_torsion_proixes( ) == 12, grm.get_n_chi_torsion_proixes()
def __init__(self,datadir,work_params,plot=False,esd_plot=False,half_data_flag=0): casetag = work_params.output.prefix # read the ground truth values back in from six.moves import cPickle as pickle # it is assumed (for now) that the reference millers contain a complete asymmetric unit # of indices, within the (d_max,d_min) region of interest and possibly outside the region. reference_millers = pickle.load(open(os.path.join(datadir,casetag+"_miller.pickle"),"rb")) experiment_manager = read_experiments(work_params) obs = pickle.load(open(os.path.join(datadir,casetag+"_observation.pickle"),"rb")) print "Read in %d observations"%(len(obs["observed_intensity"])) reference_millers.show_summary(prefix="Miller index file ") print len(obs["frame_lookup"]),len(obs["observed_intensity"]), flex.max(obs['miller_lookup']),flex.max(obs['frame_lookup']) max_frameno = flex.max(obs["frame_lookup"]) from iotbx import mtz mtz_object = mtz.object(file_name=work_params.scaling.mtz_file) #for array in mtz_object.as_miller_arrays(): # this_label = array.info().label_string() # print this_label, array.observation_type() I_sim = mtz_object.as_miller_arrays()[0].as_intensity_array() I_sim.show_summary() MODEL_REINDEX_OP = work_params.model_reindex_op I_sim = I_sim.change_basis(MODEL_REINDEX_OP).map_to_asu() #match up isomorphous (the simulated fake F's) with experimental unique set matches = miller.match_multi_indices( miller_indices_unique=reference_millers.indices(), miller_indices=I_sim.indices()) print "original unique",len(reference_millers.indices()) print "isomorphous set",len(I_sim.indices()) print "pairs",len(matches.pairs()) iso_data = flex.double(len(reference_millers.indices())) for pair in matches.pairs(): iso_data[pair[0]] = I_sim.data()[pair[1]] reference_data = miller.array(miller_set = reference_millers, data = iso_data) reference_data.set_observation_type_xray_intensity() FOBS = prepare_observations_for_scaling(work_params,obs=obs, reference_intensities=reference_data, files = experiment_manager.get_files(), half_data_flag=half_data_flag) I,I_visited,G,G_visited = I_and_G_base_estimate(FOBS,params=work_params) print "I length",len(I), "G length",len(G), "(Reference set; entire asymmetric unit)" assert len(reference_data.data()) == len(I) #presumably these assertions fail when half data are taken for CC1/2 or d_min is cut model_I = reference_data.data()[0:len(I)] T = Timer("%d frames"%(len(G), )) mapper = mapper_factory(xscale6e) minimizer = mapper(I,G,I_visited,G_visited,FOBS,params=work_params, experiments=experiment_manager.get_experiments()) del T minimizer.show_summary() Fit = minimizer.e_unpack() Gstats=flex.mean_and_variance(Fit["G"].select(G_visited==1)) print "G mean and standard deviation:",Gstats.mean(),Gstats.unweighted_sample_standard_deviation() if "Bfactor" in work_params.levmar.parameter_flags: Bstats=flex.mean_and_variance(Fit["B"].select(G_visited==1)) print "B mean and standard deviation:",Bstats.mean(),Bstats.unweighted_sample_standard_deviation() show_correlation(Fit["I"],model_I,I_visited,"Correlation of I:") Fit_stddev = minimizer.e_unpack_stddev() # XXX FIXME known bug: the length of Fit["G"] could be smaller than the length of experiment_manager.get_files() # Not sure if this has any operational drawbacks. It's a result of half-dataset selection. if plot: plot_it(Fit["I"], model_I, mode="I") if "Rxy" in work_params.levmar.parameter_flags: show_histogram(Fit["Ax"],"Histogram of x rotation (degrees)") show_histogram(Fit["Ay"],"Histogram of y rotation (degrees)") print if esd_plot: minimizer.esd_plot() from cctbx.examples.merging.show_results import show_overall_observations table1,self.n_bins,self.d_min = show_overall_observations( Fit["I"],Fit_stddev["I"],I_visited, reference_data,FOBS,title="Statistics for all reflections", work_params = work_params) self.FSIM=FOBS self.ordered_intensities=reference_data self.reference_millers=reference_millers self.Fit_I=Fit["I"] self.Fit_I_stddev=Fit_stddev["I"] self.I_visited=I_visited self.Fit = Fit self.experiments = experiment_manager
def calculate_fsc( si=None, f_array=None, # just used for binner map_coeffs=None, model_map_coeffs=None, external_map_coeffs=None, first_half_map_coeffs=None, second_half_map_coeffs=None, resolution=None, fraction_complete=None, min_fraction_complete=None, is_model_based=None, cc_cut=None, scale_using_last=None, max_cc_for_rescale=None, pseudo_likelihood=False, skip_scale_factor=False, equalize_power=False, verbose=None, out=sys.stdout): # calculate anticipated fall-off of model data with resolution if si.rmsd is None and is_model_based: si.rmsd = resolution * si.rmsd_resolution_factor print("Setting rmsd to %5.1f A based on resolution of %5.1f A" % (si.rmsd, resolution), file=out) # get f and model_f vs resolution and FSC vs resolution and apply # If external_map_coeffs then simply scale f to external_map_coeffs # scale to f_array and return sharpened map dsd = f_array.d_spacings().data() from cctbx.maptbx.segment_and_split_map import map_coeffs_to_fp if is_model_based: mc1 = map_coeffs mc2 = model_map_coeffs fo_map = map_coeffs # scale map_coeffs to model_map_coeffs*FSC fc_map = model_map_coeffs b_eff = get_b_eff(si=si, out=out) elif external_map_coeffs: mc1 = map_coeffs mc2 = external_map_coeffs fo_map = map_coeffs # scale map_coeffs to external_map_coeffs fc_map = external_map_coeffs b_eff = None else: # half_dataset mc1 = first_half_map_coeffs mc2 = second_half_map_coeffs fo_map = map_coeffs # scale map_coeffs to cc* fc_map = model_map_coeffs b_eff = None ratio_list = flex.double() target_sthol2 = flex.double() cc_list = flex.double() sthol_list = flex.double() d_min_list = flex.double() rms_fo_list = flex.double() rms_fc_list = flex.double() max_possible_cc = None n_list = flex.double() for i_bin in f_array.binner().range_used(): sel = f_array.binner().selection(i_bin) d = dsd.select(sel) if d.size() < 1: raise Sorry("Please reduce number of bins (no data in bin " + "%s) from current value of %s" % (i_bin, f_array.binner().n_bins_used())) d_min = flex.min(d) d_max = flex.max(d) d_avg = flex.mean(d) n = d.size() m1 = mc1.select(sel) m2 = mc2.select(sel) cc = m1.map_correlation(other=m2) if external_map_coeffs: cc = 1. if fo_map: fo = fo_map.select(sel) f_array_fo = map_coeffs_to_fp(fo) rms_fo = f_array_fo.data().norm() else: rms_fo = 1. if fc_map: fc = fc_map.select(sel) f_array_fc = map_coeffs_to_fp(fc) rms_fc = f_array_fc.data().norm() else: rms_fc = 1. sthol2 = 0.25 / d_avg**2 ratio_list.append(max(1.e-10, rms_fc) / max(1.e-10, rms_fo)) target_sthol2.append(sthol2) if cc is None: cc = 0. cc_list.append(cc) sthol_list.append(1 / d_avg) d_min_list.append(d_min) rms_fo_list.append(rms_fo) rms_fc_list.append(rms_fc) n_list.append(m1.size()) if b_eff is not None: max_cc_estimate = cc * math.exp(min(20., sthol2 * b_eff)) else: max_cc_estimate = cc max_cc_estimate = max(0., min(1., max_cc_estimate)) if max_possible_cc is None or (max_cc_estimate > 0 and max_cc_estimate > max_possible_cc): max_possible_cc = max_cc_estimate if verbose: print("d_min: %5.1f FC: %7.1f FOBS: %7.1f CC: %5.2f" % (d_avg, rms_fc, rms_fo, cc), file=out) if scale_using_last: # rescale to give final value average==0 cc_list, baseline = rescale_cc_list( cc_list=cc_list, scale_using_last=scale_using_last, max_cc_for_rescale=max_cc_for_rescale) if baseline is None: # don't use it scale_using_last = None original_cc_list = deepcopy(cc_list) if is_model_based: # jut smooth cc if nec fitted_cc = get_fitted_cc(cc_list=cc_list, sthol_list=sthol_list, cc_cut=cc_cut, scale_using_last=scale_using_last) cc_list = fitted_cc text = " FIT " else: cc_list = estimate_cc_star(cc_list=cc_list, sthol_list=sthol_list, cc_cut=cc_cut, scale_using_last=scale_using_last) text = " CC* " if not max_possible_cc: max_possible_cc = 0.01 if si.target_scale_factors: # not using these max_possible_cc = 1. fraction_complete = 1. elif (not is_model_based): max_possible_cc = 1. fraction_complete = 1. else: # Define overall CC based on model completeness (CC=sqrt(fraction_complete)) if fraction_complete is None: fraction_complete = max_possible_cc**2 print( "Estimated fraction complete is %5.2f based on low_res CC of %5.2f" % (fraction_complete, max_possible_cc), file=out) else: print("Using fraction complete value of %5.2f " % (fraction_complete), file=out) max_possible_cc = fraction_complete**0.5 target_scale_factors = flex.double() sum_w = 0. sum_w_scale = 0. for i_bin in f_array.binner().range_used(): index = i_bin - 1 ratio = ratio_list[index] cc = cc_list[index] sthol2 = target_sthol2[index] d_min = d_min_list[index] corrected_cc = max(0.00001, min(1., cc / max_possible_cc)) if (not is_model_based): # cc is already cc* scale_on_fo = ratio * corrected_cc elif b_eff is not None: if pseudo_likelihood: scale_on_fo = (cc / max(0.001, 1 - cc**2)) else: # usual scale_on_fo = ratio * min( 1., max(0.00001, corrected_cc) * math.exp(min(20., sthol2 * b_eff))) else: scale_on_fo = ratio * min(1., max(0.00001, corrected_cc)) w = n_list[index] * (rms_fo_list[index])**2 sum_w += w sum_w_scale += w * scale_on_fo**2 target_scale_factors.append(scale_on_fo) if not pseudo_likelihood and not skip_scale_factor: # normalize avg_scale_on_fo = (sum_w_scale / sum_w)**0.5 if equalize_power: scale_factor = 1 / avg_scale_on_fo else: # usual scale_factor = 1. / target_scale_factors.min_max_mean().max target_scale_factors=\ target_scale_factors*scale_factor print("Scale factor A: %.5f" % (scale_factor), file=out) if fraction_complete < min_fraction_complete: print("\nFraction complete (%5.2f) is less than minimum (%5.2f)..." % (fraction_complete, min_fraction_complete) + "\nSkipping scaling", file=out) target_scale_factors = flex.double(target_scale_factors.size() * (1.0, )) print("\nAverage CC: %.3f" % (cc_list.min_max_mean().mean), file=out) print("\nScale factors vs resolution:", file=out) print("Note 1: CC* estimated from sqrt(2*CC/(1+CC))", file=out) print("Note 2: CC estimated by fitting (smoothing) for values < %s" % (cc_cut), file=out) print("Note 3: Scale = A CC* rmsFc/rmsFo (A is normalization)", file=out) print(" d_min rmsFo rmsFc CC %s Scale" % (text), file=out) for sthol2, scale, rms_fo, cc, rms_fc, orig_cc in zip( target_sthol2, target_scale_factors, rms_fo_list, cc_list, rms_fc_list, original_cc_list): print("%7.2f %9.1f %9.1f %7.3f %7.3f %5.2f" % (0.5 / sthol2**0.5, rms_fo, rms_fc, orig_cc, cc, scale), file=out) si.target_scale_factors = target_scale_factors si.target_sthol2 = target_sthol2 si.d_min_list = d_min_list si.cc_list = cc_list return si
def exercise_negative_parameters(verbose=0): structure_default = xray.structure( crystal_symmetry=crystal.symmetry(unit_cell=((10, 13, 17, 75, 80, 85)), space_group_symbol="P 1"), scatterers=flex.xray_scatterer( [xray.scatterer(label="C", site=(0, 0, 0), u=0.25)])) negative_gaussian = eltbx.xray_scattering.gaussian((1, 2), (2, 3), -4) for i_trial in xrange(7): structure = structure_default.deep_copy_scatterers() scatterer = structure.scatterers()[0] if (i_trial == 1): scatterer.occupancy *= -1 elif (i_trial == 2): structure.scattering_type_registry( custom_dict={"C": negative_gaussian}) elif (i_trial == 3): scatterer.u_iso *= -1 elif (i_trial == 4): u_cart = adptbx.random_u_cart(u_scale=1, u_min=-1.1) assert max(adptbx.eigenvalues(u_cart)) < 0 u_star = adptbx.u_cart_as_u_star(structure.unit_cell(), u_cart) scatterer.u_star = u_star scatterer.flags.set_use_u_aniso_only() elif (i_trial == 5): scatterer.fp = -10 elif (i_trial == 6): scatterer.fp = -3 f_direct = structure.structure_factors(d_min=1, algorithm="direct", cos_sin_table=False).f_calc() f_fft = structure.structure_factors(d_min=1, algorithm="fft", quality_factor=1.e8, wing_cutoff=1.e-10).f_calc() if (i_trial == 2): assert negative_gaussian.at_d_star_sq( f_fft.d_star_sq().data()).all_lt(0) if (i_trial in [5, 6]): f = structure.scattering_type_registry().gaussian_not_optional( scattering_type="C").at_d_star_sq(f_fft.d_star_sq().data()) if (i_trial == 5): assert flex.max(f) + scatterer.fp < 0 else: assert flex.max(f) + scatterer.fp > 0 assert flex.min(f) + scatterer.fp < 0 cc = flex.linear_correlation(abs(f_direct).data(), abs(f_fft).data()).coefficient() if (cc < 0.999): raise AssertionError("i_trial=%d, correlation=%.6g" % (i_trial, cc)) elif (0 or verbose): print "correlation=%.6g" % cc # # very simple test of gradient calculations with negative parameters structure_factor_gradients = \ cctbx.xray.structure_factors.gradients( miller_set=f_direct, cos_sin_table=False) target_functor = xray.target_functors.intensity_correlation( f_obs=abs(f_direct)) target_result = target_functor(f_fft, True) xray.set_scatterer_grad_flags(scatterers=structure.scatterers(), site=True, u_iso=True, u_aniso=True, occupancy=True, fp=True, fdp=True) for algorithm in ["direct", "fft"]: grads = structure_factor_gradients( xray_structure=structure, u_iso_refinable_params=None, miller_set=f_direct, d_target_d_f_calc=target_result.derivatives(), n_parameters=structure.n_parameters(), algorithm=algorithm).packed()
def run(): import sys reflection_file_name = sys.argv[1] import iotbx.cif miller_arrays = iotbx.cif.reader( file_path=reflection_file_name).as_miller_arrays() for miller_array in miller_arrays: s = str(miller_array.info()) if '_meas' in s: if miller_array.is_xray_intensity_array(): break elif miller_array.is_xray_amplitude_array(): break if not ('_meas' in str(miller_array.info()) and (miller_array.is_xray_amplitude_array() or miller_array.is_xray_intensity_array())): print("Sorry: CIF does not contain an appropriate miller array") return miller_array.show_comprehensive_summary() print() if (miller_array.is_xray_intensity_array()): print("Converting intensities to amplitudes.") miller_array = miller_array.as_amplitude_array() print() miller_array.setup_binner(auto_binning=True) miller_array.binner().show_summary() print() all_e_values = miller_array.quasi_normalize_structure_factors().sort( by_value="data") large_e_values = all_e_values.select(all_e_values.data() > 1.2) print("number of large_e_values:", large_e_values.size()) print() from cctbx import dmtbx triplets = dmtbx.triplet_generator(large_e_values) from cctbx.array_family import flex print("triplets per reflection: min,max,mean: %d, %d, %.2f" % (flex.min(triplets.n_relations()), flex.max(triplets.n_relations()), flex.mean(triplets.n_relations().as_double()))) print("total number of triplets:", flex.sum(triplets.n_relations())) print() input_phases = large_e_values \ .random_phases_compatible_with_phase_restrictions() tangent_formula_phases = input_phases.data() for i in range(10): tangent_formula_phases = triplets.apply_tangent_formula( amplitudes=large_e_values.data(), phases_rad=tangent_formula_phases, selection_fixed=None, use_fixed_only=False, reuse_results=True) e_map_coeff = large_e_values.phase_transfer( phase_source=tangent_formula_phases) from cctbx import maptbx e_map = e_map_coeff.fft_map(symmetry_flags=maptbx.use_space_group_symmetry) e_map.apply_sigma_scaling() e_map.statistics().show_summary(prefix="e_map ") print() peak_search = e_map.peak_search(parameters=maptbx.peak_search_parameters( min_distance_sym_equiv=1.2)) peaks = peak_search.all(max_clusters=10) print("e_map peak list") print(" fractional coordinates peak height") for site, height in zip(peaks.sites(), peaks.heights()): print(" (%9.6f, %9.6f, %9.6f)" % site, "%10.3f" % height) print() if (len(sys.argv) > 2): coordinate_file_name = sys.argv[2] xray_structure = iotbx.cif.reader( file_path=coordinate_file_name).build_crystal_structures( data_block_name="I") xray_structure.show_summary().show_scatterers() print() f_calc = abs( miller_array.structure_factors_from_scatterers( xray_structure=xray_structure, algorithm="direct").f_calc()) correlation = flex.linear_correlation(f_calc.data(), miller_array.data()) assert correlation.is_well_defined() print("correlation of f_obs and f_calc: %.4f" % correlation.coefficient()) print() reference_model = xray_structure.as_emma_model() assert reference_model.unit_cell().is_similar_to(e_map.unit_cell()) assert reference_model.space_group() == e_map.space_group() from cctbx import euclidean_model_matching as emma peak_model = emma.model(special_position_settings=reference_model) for i, site in enumerate(peaks.sites()): peak_model.add_position( emma.position(label="peak%02d" % i, site=site)) matches = emma.model_matches( model1=reference_model, model2=peak_model, tolerance=1., models_are_diffraction_index_equivalent=True) for match in matches.refined_matches: match.show()
def exercise_1(): #------------------------------------------------------------------------TEST-1 if (1): r = (1.67, 0.68, 0.0, 0.5, -0.31, -0.64, -1.63) d = 2.0 * van_der_waals_radii.vdw.table["C"] a = (d / 2.) * (4. / 3. * math.pi * 2.)**(1. / 3.) for x in r: for y in r: for z in r: structure, xyzf, atmrad = structure_init( (x, y, z), "P1", (a, a, a, 90, 90, 90)) step = 0.1 crystal_gridding = maptbx.crystal_gridding( unit_cell=structure.unit_cell(), step=step) shrink_truncation_radius = 0.0 solvent_radius = 0.0 m = around_atoms(structure.unit_cell(), structure.space_group().order_z(), xyzf, atmrad, crystal_gridding.n_real(), solvent_radius, shrink_truncation_radius) assert m.solvent_radius == 0 assert m.shrink_truncation_radius == 0 assert approx_equal(1. / m.accessible_surface_fraction, 2.0, 1e-2) assert approx_equal(1. / m.contact_surface_fraction, 2.0, 1e-2) assert flex.max(m.data) == 1 assert flex.min(m.data) == 0 assert m.data.size() == m.data.count(1) + m.data.count(0) #------------------------------------------------------------------------TEST-2 if (1): r = (-2.0, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0) asf = [] csf = [] for x in r: for y in r: for z in r: structure, xyzf, atmrad = structure_init( (x, y, z), "C2", (5.0, 5.5, 7.0, 90, 80, 90)) step = 0.25 crystal_gridding = maptbx.crystal_gridding( unit_cell=structure.unit_cell(), step=step) shrink_truncation_radius = 0.0 solvent_radius = 0.0 m = around_atoms(structure.unit_cell(), structure.space_group().order_z(), xyzf, atmrad, crystal_gridding.n_real(), solvent_radius, shrink_truncation_radius) asf.append(m.accessible_surface_fraction) csf.append(m.contact_surface_fraction) assert m.accessible_surface_fraction == m.contact_surface_fraction assert approx_equal(min(asf), max(asf)) #------------------------------------------------------------------------TEST-3 if (1): r = (1.673, 0.685, -0.315, -0.649, -1.637) asf = [] csf = [] for x in r: for y in r: for z in r: structure, xyzf, atmrad = structure_init( (x, y, z), "P1", (5.0, 5.5, 6.0, 70, 80, 100)) step = 0.1 crystal_gridding = maptbx.crystal_gridding( unit_cell=structure.unit_cell(), step=step) shrink_truncation_radius = 0.0 solvent_radius = 0.0 m = around_atoms(structure.unit_cell(), structure.space_group().order_z(), xyzf, atmrad, crystal_gridding.n_real(), solvent_radius, shrink_truncation_radius) asf.append(m.accessible_surface_fraction) csf.append(m.contact_surface_fraction) assert m.accessible_surface_fraction == m.contact_surface_fraction assert approx_equal(min(asf), max(asf), 1e-3)
def fit_side_chain(self, clusters): rotamer_iterator = \ mmtbx.refinement.real_space.fit_residue.get_rotamer_iterator( mon_lib_srv = self.mon_lib_srv, residue = self.residue) if (rotamer_iterator is None): return #selection_rsr = flex.size_t(flatten(clusters[0].vector)) selection_clash = self.co.clash_eval_selection selection_rsr = self.co.rsr_eval_selection if (self.target_map is not None): start_target_value = self.get_target_value( sites_cart=self.residue.atoms().extract_xyz(), selection=selection_rsr) sites_cart_start = self.residue.atoms().extract_xyz() sites_cart_first_rotamer = list(rotamer_iterator)[0][1] self.residue.atoms().set_xyz(sites_cart_first_rotamer) axes = [] atr = [] for i, angle in enumerate(self.chi_angles[0]): cl = clusters[i] axes.append(flex.size_t(cl.axis)) atr.append(flex.size_t(cl.atoms_to_rotate)) sites = self.residue.atoms().extract_xyz() if (self.target_map is not None and self.xyzrad_bumpers is not None): # Get vdW radii radii = flex.double() atom_names = [] for a in self.residue.atoms(): atom_names.append(a.name.strip()) converter = iotbx.pdb.residue_name_plus_atom_names_interpreter( residue_name=self.residue.resname, atom_names=atom_names) mon_lib_names = converter.atom_name_interpretation.mon_lib_names() for n in mon_lib_names: try: radii.append(self.vdw_radii[n.strip()] - 0.25) except KeyError: radii.append(1.5) # XXX U, Uranium, OXT are problems! # xyzrad_residue = ext.xyzrad(sites_cart=sites, radii=radii) # ro = ext.fit(target_value=start_target_value, xyzrad_bumpers=self.xyzrad_bumpers, axes=axes, rotatable_points_indices=atr, angles_array=self.chi_angles, density_map=self.target_map, all_points=xyzrad_residue, unit_cell=self.unit_cell, selection_clash=selection_clash, selection_rsr=selection_rsr, sin_table=self.sin_cos_table.sin_table, cos_table=self.sin_cos_table.cos_table, step=self.sin_cos_table.step, n=self.sin_cos_table.n) elif (self.target_map is not None and self.xyzrad_bumpers is None): ro = ext.fit(target_value=start_target_value, axes=axes, rotatable_points_indices=atr, angles_array=self.chi_angles, density_map=self.target_map, all_points=sites, unit_cell=self.unit_cell, selection=selection_rsr, sin_table=self.sin_cos_table.sin_table, cos_table=self.sin_cos_table.cos_table, step=self.sin_cos_table.step, n=self.sin_cos_table.n) else: ro = ext.fit(sites_cart_start=sites_cart_start.deep_copy(), axes=axes, rotatable_points_indices=atr, angles_array=self.chi_angles, all_points=self.residue.atoms().extract_xyz(), sin_table=self.sin_cos_table.sin_table, cos_table=self.sin_cos_table.cos_table, step=self.sin_cos_table.step, n=self.sin_cos_table.n) sites_cart_result = ro.result() if (sites_cart_result.size() > 0): dist = None if (self.accept_only_if_max_shift_is_smaller_than is not None): dist = flex.max( flex.sqrt((sites_cart_start - sites_cart_result).dot())) if (dist is None): self.residue.atoms().set_xyz(sites_cart_result) else: if (dist is not None and dist < self.accept_only_if_max_shift_is_smaller_than): self.residue.atoms().set_xyz(sites_cart_result) else: self.residue.atoms().set_xyz(sites_cart_start) else: self.residue.atoms().set_xyz(sites_cart_start)
def select_best_by_cc_mproc(shot_no, cryst_id, iparams): if iparams.iotacc.set_id is not None: data_dir = iparams.data[0] + "/" + iparams.iotacc.set_id else: data_dir = iparams.data[0] cryst_id_shot_no = (cryst_id + "_0_" + str(shot_no).zfill(iparams.iotacc.LEN_SHOT_SEQ)) # cryst_id_shot_no = cryst_id+str(shot_no).zfill(LEN_SHOT_SEQ) cc_list = flex.double() n_refl_list = flex.int() pickle_filename_list = [] flag_success = False d_min, d_max = (iparams.iotacc.d_min, iparams.iotacc.d_max) txt_out = ("ID".center(3) + "shot".center(6) + "res.".center(7) + "n_refl".center(7) + "cc_pt".center(7) + "cc_fl".center(7) + "n_ref".center(7) + "cc_pt".center(7) + "cc_fl".center(7) + "n_iso".center(7) + "a".center(7) + "b".center(7) + "c".center(7) + "alp".center(7) + "gam".center(7) + "bet".center(7) + "file name".center(20) + "\n") for tmp_shot_dir in os.listdir(data_dir + "/" + cryst_id): if tmp_shot_dir.find(cryst_id_shot_no) > 0: data = data_dir + "/" + cryst_id + "/" + tmp_shot_dir # data = data_dir+'/'+tmp_shot_dir frame_files = [] if os.path.isdir(data): frame_files = read_pickles(data) if len(frame_files) == 0: print(data, " - no pickle file found.") else: for pickle_filename in frame_files: pickle_filename_split = pickle_filename.split("/") pickle_filename_only = pickle_filename_split[ len(pickle_filename_split) - 1] pickle_filename_only_split = pickle_filename_only.split( "_") observations_Eoc_corrected, observations_original = get_Eoc_corrected_observations( pickle_filename, iparams) # check unitcell uc_params = observations_Eoc_corrected.unit_cell( ).parameters() cc_eoc, n_refl_eoc, cc_ori, n_refl_ori, cc_iso_eoc, n_refl_iso_eoc, cc_iso_ori, n_refl_iso_ori = ( 0, 0, 0, 0, 0, 0, 0, 0, ) a, b, c, alpha, beta, gamma = uc_params if good_unit_cell( uc_params, iparams.target_unit_cell.parameters(), iparams.iotacc.uc_tolerance, ): try: miller_set = symmetry( unit_cell=observations_Eoc_corrected.unit_cell( ).parameters(), space_group_symbol=str( iparams.target_space_group), ).build_miller_set( anomalous_flag=iparams.target_anomalous_flag, d_min=d_min, ) observations_Eoc_corrected = observations_Eoc_corrected.customized_copy( anomalous_flag=iparams.target_anomalous_flag, crystal_symmetry=miller_set.crystal_symmetry(), ) miller_set = symmetry( unit_cell=observations_original.unit_cell(). parameters(), space_group_symbol=str( iparams.target_space_group), ).build_miller_set( anomalous_flag=iparams.target_anomalous_flag, d_min=d_min, ) observations_original = observations_original.customized_copy( anomalous_flag=iparams.target_anomalous_flag, crystal_symmetry=miller_set.crystal_symmetry(), ) # convert to asymmetric unit index observations_Eoc_corrected_asu = ( observations_Eoc_corrected.map_to_asu()) observations_original_asu = ( observations_original.map_to_asu()) # calculate CCori CCEoc miller_array_ref = get_observations_ref( iparams.hklrefin) cc_eoc, n_refl_eoc = calc_cc( miller_array_ref, observations_Eoc_corrected_asu) cc_ori, n_refl_ori = calc_cc( miller_array_ref, observations_original_asu) miller_array_iso = get_observations_ref( iparams.hklisoin) cc_iso_eoc, n_refl_iso_eoc = calc_cc( miller_array_iso, observations_Eoc_corrected_asu) cc_iso_ori, n_refl_iso_ori = calc_cc( miller_array_iso, observations_original_asu) except Exception: dummy = 0 # print cryst_id, shot_no, sel_type.ljust(15,' '), 'mismatch spacegroup %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f'%(a,b,c,alpha,beta,gamma) if cc_eoc > 0: pickle_filename_list.append(pickle_filename) cc_list.append(cc_eoc) n_refl_list.append(n_refl_eoc) txt_out += ( cryst_id + " " + str(shot_no).zfill(iparams.iotacc.LEN_SHOT_SEQ) + " %6.2f %6.0f %6.2f %6.2f %6.0f %6.2f %6.2f %6.0f %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f" % ( observations_original.d_min(), len(observations_original.data()), cc_ori, cc_eoc, n_refl_eoc, cc_iso_ori, cc_iso_eoc, n_refl_iso_eoc, a, b, c, alpha, beta, gamma, ) + " " + pickle_filename_only + "\n") # select and copy best_by_cc shot if len(pickle_filename_list) > 0: # sort by cc and select best n results cc_thres = flex.max(cc_list) - (flex.max(cc_list) * iparams.iotacc.percent_top_cc / 100) i_sel = cc_list > cc_thres cc_list_sel = cc_list.select(i_sel) n_refl_list_sel = n_refl_list.select(i_sel) i_seq_sel = flex.int(range(len(cc_list))).select(i_sel) pickle_filename_list_sel = [ pickle_filename_list[ii_sel] for ii_sel in i_seq_sel ] # sort by n_refl and select the first i_sorted = flex.sort_permutation(n_refl_list_sel, reverse=True) n_refl_sel = n_refl_list_sel[i_sorted[0]] cc_sel = cc_list_sel[i_sorted[0]] pickle_filename_sel = pickle_filename_list_sel[i_sorted[0]] pickle_filename_sel_split = pickle_filename_sel.split("/") pickle_filename_sel_only = pickle_filename_sel_split[ len(pickle_filename_sel_split) - 1] pickle_filename_out = (iparams.run_no + "/" + iparams.iotacc.set_id + "_" + pickle_filename_sel_only) # shutil.copyfile(pickle_filename_sel, pickle_filename_out) txt_out += ("Select --> CC_thres=%6.2f Best CC=%6.2f N_refl=%4.0f " % (cc_thres, cc_sel, n_refl_sel) + pickle_filename_out + "\n") flag_success = True else: txt_out += "No image with CC > 0\n" if flag_success: return pickle_filename_sel, txt_out else: return None
def plot_stats(self, results, iparams): #retrieve stats from results and plot them if iparams.flag_plot or iparams.flag_output_verbose: #for plotting set n_bins = 5 to avoid empty bin n_bins_plot = 5 #get expected f^2 try: mxh = mx_handler() asu_contents = mxh.get_asu_contents(iparams.n_residues) observations_as_f = results[0].observations.as_amplitude_array( ) binner_template_asu = observations_as_f.setup_binner( n_bins=n_bins_plot) wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True) expected_f_sq = wp.expected_f_sq mean_stol_sq = wp.mean_stol_sq except Exception: expected_f_sq = flex.double([0] * n_bins_plot) mean_stol_sq = flex.double(range(n_bins_plot)) print "Warning: Wilson plot calculation in plot stats failed." #setup list params_array = np.array([[pres.R_init, pres.R_final, pres.R_xy_init, pres.R_xy_final, \ pres.G, pres.B, pres.rotx*180/math.pi, pres.roty*180/math.pi, \ pres.ry, pres.rz, pres.r0, pres.re, pres.voigt_nu, \ pres.uc_params[0], pres.uc_params[1], pres.uc_params[2], \ pres.uc_params[3], pres.uc_params[4], pres.uc_params[5], \ pres.CC_final, pres.pickle_filename] for pres in results]) params = ['Rinit','Rfinal','Rxyinit', 'Rxyfinal', \ 'G','B','rot_x','rot_y','gamma_y','gamma_z','gamma_0','gamma_e','voigtnu' , \ 'a','b','c','alpha','beta','gamma','CC','Filename'] #keep parameter history if verbose is selected if iparams.flag_output_verbose: fileseq_list = flex.int() for file_in in os.listdir(iparams.run_no): if file_in.endswith('.paramhist'): file_split = file_in.split('.') fileseq_list.append(int(file_split[0])) if len(fileseq_list) == 0: new_fileseq = 0 else: new_fileseq = flex.max(fileseq_list) + 1 newfile_name = str(new_fileseq) + '.paramhist' txt_out_verbose = '\n'.join([' '.join(p) for p in params_array]) f = open(iparams.run_no + '/' + newfile_name, 'w') f.write(txt_out_verbose) f.close() #plotting if iparams.flag_plot: n_rows = 3 n_cols = int(math.ceil(len(params) / n_rows)) num_bins = 10 for i in xrange(len(params) - 1): tmp_params = params_array[:, i].astype(np.float) plt.subplot(n_rows, n_cols, i + 1) plt.hist(tmp_params, num_bins, normed=0, facecolor='green', alpha=0.5) plt.ylabel('Frequencies') plt.title(params[i] + '\nmu %5.1f med %5.1f sigma %5.1f' % (np.mean(tmp_params), np.median(tmp_params), np.std(tmp_params))) plt.show()
def __init__(self, **kwargs): group_args.__init__(self, **kwargs) print('finished Dij, now calculating rho_i and density') from xfel.clustering import Rodriguez_Laio_clustering_2014 as RL R = RL(distance_matrix=self.Dij, d_c=self.d_c) #from IPython import embed; embed(); exit() #from clustering.plot_with_dimensional_embedding import plot_with_dimensional_embedding #plot_with_dimensional_embedding(1-self.Dij/flex.max(self.Dij), show_plot=True) self.rho = rho = R.get_rho() ave_rho = flex.mean(rho.as_double()) NN = self.Dij.focus()[0] i_max = flex.max_index(rho) delta_i_max = flex.max( flex.double([self.Dij[i_max, j] for j in range(NN)])) rho_order = flex.sort_permutation(rho, reverse=True) rho_order_list = list(rho_order) self.delta = delta = R.get_delta(rho_order=rho_order, delta_i_max=delta_i_max) cluster_id = flex.int(NN, -1) # -1 means no cluster delta_order = flex.sort_permutation(delta, reverse=True) MAX_PERCENTILE_RHO = self.max_percentile_rho # cluster centers have to be in the top percentile n_cluster = 0 # pick_top_solution = False rho_stdev = flex.mean_and_variance( rho.as_double()).unweighted_sample_standard_deviation() delta_stdev = flex.mean_and_variance( delta).unweighted_sample_standard_deviation() if rho_stdev != 0.0 and delta_stdev != 0: rho_z = (rho.as_double() - flex.mean(rho.as_double())) / (rho_stdev) delta_z = (delta - flex.mean(delta)) / (delta_stdev) else: pick_top_solution = True if rho_stdev == 0.0: centroids = [flex.first_index(delta, flex.max(delta))] elif delta_stdev == 0.0: centroids = [flex.first_index(rho, flex.max(rho))] significant_delta = [] significant_rho = [] debug_fix_clustering = True if debug_fix_clustering: if not pick_top_solution: delta_z_cutoff = min(1.0, max(delta_z)) rho_z_cutoff = min(1.0, max(rho_z)) for ic in range(NN): # test the density & rho if delta_z[ic] >= delta_z_cutoff: significant_delta.append(ic) if rho_z[ic] >= rho_z_cutoff: significant_rho.append(ic) centroid_candidates = list( set(significant_delta).intersection(set(significant_rho))) # Now compare the relative orders of the max delta_z and max rho_z to make sure they are within 1 stdev centroids = [] max_delta_z_candidates = -999.9 max_rho_z_candidates = -999.9 for ic in centroid_candidates: if delta_z[ic] > max_delta_z_candidates: max_delta_z_candidates = delta_z[ic] if rho_z[ic] > max_rho_z_candidates: max_rho_z_candidates = rho_z[ic] for ic in centroid_candidates: if max_delta_z_candidates - delta_z[ ic] < 1.0 and max_rho_z_candidates - rho_z[ ic] < 1.0: centroids.append(ic) item_idxs = [ delta_order[ic] for ic, centroid in enumerate(centroids) ] for item_idx in item_idxs: cluster_id[item_idx] = n_cluster print('CLUSTERING_STATS', item_idx, cluster_id[item_idx]) n_cluster += 1 #### else: for ic in range(NN): item_idx = delta_order[ic] if ic != 0: if delta[item_idx] <= 0.25 * delta[ delta_order[0]]: # too low to be a medoid continue item_rho_order = rho_order_list.index(item_idx) if (item_rho_order) / NN < MAX_PERCENTILE_RHO: cluster_id[item_idx] = n_cluster print('CLUSTERING_STATS', ic, item_idx, item_rho_order, cluster_id[item_idx]) n_cluster += 1 ### # # print('Found %d clusters' % n_cluster) for x in range(NN): if cluster_id[x] >= 0: print("XC", x, cluster_id[x], rho[x], delta[x]) self.cluster_id_maxima = cluster_id.deep_copy() R.cluster_assignment(rho_order, cluster_id) self.cluster_id_full = cluster_id.deep_copy() #halo = flex.bool(NN,False) #border = R.get_border( cluster_id = cluster_id ) #for ic in range(n_cluster): #loop thru all border regions; find highest density # this_border = (cluster_id == ic) & (border==True) # if this_border.count(True)>0: # highest_density = flex.max(rho.select(this_border)) # halo_selection = (rho < highest_density) & (this_border==True) # if halo_selection.count(True)>0: # cluster_id.set_selected(halo_selection,-1) # core_selection = (cluster_id == ic) & ~halo_selection # highest_density = flex.max(rho.select(core_selection)) # too_sparse = core_selection & (rho.as_double() < highest_density/10.) # another heuristic # if too_sparse.count(True)>0: # cluster_id.set_selected(too_sparse,-1) self.cluster_id_final = cluster_id.deep_copy()
def run(args, log=None, ccp4_map=None, return_as_miller_arrays=False, nohl=False, space_group_number=None, out=sys.stdout): if log is None: log = out inputs = mmtbx.utils.process_command_line_args( args=args, master_params=master_params()) got_map = False if ccp4_map: got_map = True broadcast(m="Parameters:", log=log) inputs.params.show(prefix=" ", out=out) params = inputs.params.extract() if (ccp4_map is None and inputs.ccp4_map is not None): broadcast(m="Processing input CCP4 map file: %s" % inputs.ccp4_map_file_name, log=log) ccp4_map = inputs.ccp4_map ccp4_map.show_summary(prefix=" ", out=out) got_map = True if (not got_map): raise Sorry("Map file is needed.") # m = ccp4_map if (m.space_group_number > 1): raise Sorry("Input map space group: %d. Must be P1." % m.space_group_number) broadcast(m="Input map information:", log=log) print >> out, "m.all() :", m.data.all() print >> out, "m.focus() :", m.data.focus() print >> out, "m.origin():", m.data.origin() print >> out, "m.nd() :", m.data.nd() print >> out, "m.size() :", m.data.size() print >> out, "m.focus_size_1d():", m.data.focus_size_1d() print >> out, "m.is_0_based() :", m.data.is_0_based() print >> out, "map: min/max/mean:", flex.min(m.data), flex.max( m.data), flex.mean(m.data) print >> out, "unit cell:", m.unit_cell_parameters # map_data = m.data shift_needed = not \ (map_data.focus_size_1d() > 0 and map_data.nd() == 3 and map_data.is_0_based()) if (shift_needed): map_data = map_data.shift_origin() # generate complete set of Miller indices up to given high resolution d_min n_real = map_data.focus() if not space_group_number: space_group_number = 1 if space_group_number <= 1: symmetry_flags = None else: symmetry_flags = maptbx.use_space_group_symmetry, cs = crystal.symmetry(m.unit_cell_parameters, space_group_number) crystal_gridding = maptbx.crystal_gridding( unit_cell=cs.unit_cell(), space_group_info=cs.space_group_info(), symmetry_flags=symmetry_flags, pre_determined_n_real=n_real) # d_min = params.d_min if (d_min is None and not params.box): d_min = maptbx.d_min_from_map(map_data=map_data, unit_cell=cs.unit_cell()) if (d_min is None): # box of reflections in |h|<N1/2, |k|<N2/2, 0<=|l|<N3/2 max_index = [(i - 1) // 2 for i in n_real] print >> out, "max_index:", max_index complete_set = miller.build_set(crystal_symmetry=cs, anomalous_flag=False, max_index=max_index) indices = complete_set.indices() indices.append((0, 0, 0)) complete_set = complete_set.customized_copy(indices=indices) #if(not params.box): # # XXX What is sphere resolution corresponding to given box? # uc = complete_set.unit_cell() # d1 = uc.d([0,0,max_index[2]]) # d2 = uc.d([0,max_index[1],0]) # d3 = uc.d([max_index[0],1,0]) # print >>out, d1,d2,d3 # complete_set_sp = miller.build_set( # crystal_symmetry = cs, # anomalous_flag = False, # d_min = min(d1,d2,d3)) # complete_set = complete_set.common_set(complete_set_sp) else: complete_set = miller.build_set(crystal_symmetry=cs, anomalous_flag=False, d_min=d_min) broadcast(m="Complete set information:", log=log) complete_set.show_comprehensive_summary(prefix=" ", f=out) try: f_obs_cmpl = complete_set.structure_factors_from_map( map=map_data.as_double(), use_scale=True, anomalous_flag=False, use_sg=False) except Exception, e: if (str(e) == "cctbx Error: Miller index not in structure factor map." ): msg = "Too high resolution requested. Try running with larger d_min." raise Sorry(msg) else: raise Sorry(str(e))
def exercise(space_group_info, const_gaussian, negative_gaussian, anomalous_flag, allow_mix, use_u_iso, use_u_aniso, d_min=1., resolution_factor=1. / 3, max_prime=5, quality_factor=100, wing_cutoff=1.e-6, exp_table_one_over_step_size=-100, force_complex=False, verbose=0): if (const_gaussian): elements = ["const"] * 8 elif (negative_gaussian): elements = ["H"] * 8 else: elements = ["N", "C", "C", "O", "N", "C", "C", "O"] if (random.random() < 0.5): random_f_prime_scale = 0.6 else: random_f_prime_scale = 0 structure = random_structure.xray_structure( space_group_info, elements=elements, random_f_prime_d_min=1, random_f_prime_scale=random_f_prime_scale, random_f_double_prime=anomalous_flag, use_u_aniso=True, use_u_iso=False, random_u_cart_scale=0.3, random_u_iso=True, random_occupancy=True) random_structure.random_modify_adp_and_adp_flags_2( scatterers=structure.scatterers(), use_u_iso=use_u_iso, use_u_aniso=use_u_aniso, allow_mix=allow_mix, random_u_iso_scale=0.3, random_u_iso_min=0.0) sampled_density_must_be_positive = True if (negative_gaussian): reg = structure.scattering_type_registry( custom_dict={"H": eltbx.xray_scattering.gaussian(-1)}) assert reg.gaussian("H").n_terms() == 0 assert reg.gaussian("H").c() == -1 sampled_density_must_be_positive = False elif (not const_gaussian and random.random() < 0.5): if (random.random() < 0.5): sampled_density_must_be_positive = False assign_custom_gaussians( structure, negative_a=not sampled_density_must_be_positive) f_direct = structure.structure_factors(anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct").f_calc() crystal_gridding = f_direct.crystal_gridding( resolution_factor=resolution_factor, d_min=d_min, max_prime=max_prime) assert crystal_gridding.symmetry_flags() is None rfft = fftpack.real_to_complex_3d(crystal_gridding.n_real()) u_base = xray.calc_u_base(d_min, resolution_factor, quality_factor) omptbx.env.num_threads = libtbx.introspection.number_of_processors() sampled_density = xray.sampled_model_density( unit_cell=structure.unit_cell(), scatterers=structure.scatterers(), scattering_type_registry=structure.scattering_type_registry(), fft_n_real=rfft.n_real(), fft_m_real=rfft.m_real(), u_base=u_base, wing_cutoff=wing_cutoff, exp_table_one_over_step_size=exp_table_one_over_step_size, force_complex=force_complex, sampled_density_must_be_positive=sampled_density_must_be_positive, tolerance_positive_definite=1.e-5, use_u_base_as_u_extra=False) focus = sampled_density.real_map_unpadded().focus() all = sampled_density.real_map_unpadded().all() last = sampled_density.real_map_unpadded().last() assert approx_equal(focus, last) assert approx_equal(all, last) assert sampled_density.anomalous_flag() == (anomalous_flag or force_complex) if (0 or verbose): print "const_gaussian:", const_gaussian print "negative_gaussian:", negative_gaussian print "number of scatterers passed:", \ sampled_density.n_scatterers_passed() print "number of contributing scatterers:", \ sampled_density.n_contributing_scatterers() print "number of anomalous scatterers:", \ sampled_density.n_anomalous_scatterers() print "wing_cutoff:", sampled_density.wing_cutoff() print "exp_table_one_over_step_size:", \ sampled_density.exp_table_one_over_step_size() print "exp_table_size:", sampled_density.exp_table_size() print "max_sampling_box_edges:", sampled_density.max_sampling_box_edges( ), print "(%.4f, %.4f, %.4f)" % sampled_density.max_sampling_box_edges_frac( ) if (not sampled_density.anomalous_flag()): print "map min:", flex.min(sampled_density.real_map()) print "map max:", flex.max(sampled_density.real_map()) else: print "map min:", flex.min(flex.real( sampled_density.complex_map())), print flex.min(flex.imag(sampled_density.complex_map())) print "map max:", flex.max(flex.real( sampled_density.complex_map())), print flex.max(flex.imag(sampled_density.complex_map())) if (not sampled_density.anomalous_flag() and negative_gaussian): assert flex.min(sampled_density.real_map()) < 0 assert flex.max(sampled_density.real_map()) == 0 if (not sampled_density.anomalous_flag()): map = sampled_density.real_map() assert map.all() == rfft.m_real() assert map.focus() == rfft.n_real() sf_map = rfft.forward(map) assert sf_map.all() == rfft.n_complex() assert sf_map.focus() == rfft.n_complex() collect_conj = True else: cfft = fftpack.complex_to_complex_3d(rfft.n_real()) map = sampled_density.complex_map() assert map.all() == cfft.n() assert map.focus() == cfft.n() sf_map = cfft.backward(map) assert sf_map.all() == cfft.n() assert sf_map.focus() == cfft.n() collect_conj = False f_fft_data = maptbx.structure_factors.from_map( space_group=f_direct.space_group(), anomalous_flag=sampled_density.anomalous_flag(), miller_indices=f_direct.indices(), complex_map=sf_map, conjugate_flag=collect_conj).data() sampled_density.eliminate_u_extra_and_normalize(f_direct.indices(), f_fft_data) structure_factor_utils.check_correlation("direct/fft_regression", f_direct.indices(), 0, f_direct.data(), f_fft_data, min_corr_ampl=1 * 0.99, max_mean_w_phase_error=1 * 3., verbose=verbose) f_fft = xray.structure_factors.from_scatterers( miller_set=f_direct, grid_resolution_factor=resolution_factor, quality_factor=quality_factor, wing_cutoff=wing_cutoff, exp_table_one_over_step_size=exp_table_one_over_step_size, sampled_density_must_be_positive=sampled_density_must_be_positive, max_prime=max_prime)(xray_structure=structure, miller_set=f_direct, algorithm="fft").f_calc() structure_factor_utils.check_correlation("direct/fft_xray", f_direct.indices(), 0, f_direct.data(), f_fft.data(), min_corr_ampl=1 * 0.99, max_mean_w_phase_error=1 * 3., verbose=verbose)
def __init__(self, **kwargs): group_args.__init__(self, **kwargs) # require Dij, d_c P = Profiler("2. calculate rho density") print("finished Dij, now calculating rho_i, the density") from xfel.clustering import Rodriguez_Laio_clustering_2014 # alternative clustering algorithms: see http://scikit-learn.org/stable/modules/clustering.html # also see https://cran.r-project.org/web/packages/dbscan/vignettes/hdbscan.html # see also https://en.wikipedia.org/wiki/Hausdorff_dimension R = Rodriguez_Laio_clustering_2014(distance_matrix=self.Dij, d_c=self.d_c) self.rho = rho = R.get_rho() ave_rho = flex.mean(rho.as_double()) NN = self.Dij.focus()[0] print("The average rho_i is %5.2f, or %4.1f%%" % (ave_rho, 100 * ave_rho / NN)) i_max = flex.max_index(rho) P = Profiler("3.transition") print("the index with the highest density is %d" % (i_max)) delta_i_max = flex.max( flex.double([self.Dij[i_max, j] for j in range(NN)])) print("delta_i_max", delta_i_max) rho_order = flex.sort_permutation(rho, reverse=True) rho_order_list = list(rho_order) P = Profiler("4. delta") self.delta = delta = R.get_delta(rho_order=rho_order, delta_i_max=delta_i_max) P = Profiler("5. find cluster maxima") #---- Now hunting for clusters ---Lot's of room for improvement (or simplification) here!!! cluster_id = flex.int(NN, -1) # default -1 means no cluster delta_order = flex.sort_permutation(delta, reverse=True) N_CLUST = 10 # maximum of 10 points to be considered as possible clusters #MAX_PERCENTILE_DELTA = 0.99 # cluster centers have to be in the top 10% percentile delta MAX_PERCENTILE_RHO = 0.99 # cluster centers have to be in the top 75% percentile rho n_cluster = 0 #max_n_delta = min(N_CLUST, int(MAX_PERCENTILE_DELTA*NN)) for ic in range(NN): # test the density, rho item_idx = delta_order[ic] if delta[item_idx] > 100: print("A: iteration", ic, "delta", delta[item_idx], delta[item_idx] < 0.25 * delta[delta_order[0]]) if delta[item_idx] < 0.25 * delta[ delta_order[0]]: # too low (another heuristic!) continue item_rho_order = rho_order_list.index(item_idx) if delta[item_idx] > 100: print("B: iteration", ic, item_rho_order, item_rho_order / NN, MAX_PERCENTILE_RHO) if item_rho_order / NN < MAX_PERCENTILE_RHO: cluster_id[item_idx] = n_cluster print(ic, item_idx, item_rho_order, cluster_id[item_idx]) n_cluster += 1 print("Found %d clusters" % n_cluster) for x in range(NN): if cluster_id[x] >= 0: print("XC", x, cluster_id[x], rho[x], delta[x]) self.cluster_id_maxima = cluster_id.deep_copy() P = Profiler("6. assign all points") R.cluster_assignment(rho_order, cluster_id) self.cluster_id_full = cluster_id.deep_copy() # assign the halos P = Profiler("7. assign halos") halo = flex.bool(NN, False) border = R.get_border(cluster_id=cluster_id) for ic in range(n_cluster ): #loop thru all border regions; find highest density print("cluster", ic, "in border", border.count(True)) this_border = (cluster_id == ic) & (border == True) print(len(this_border), this_border.count(True)) if this_border.count(True) > 0: highest_density = flex.max(rho.select(this_border)) halo_selection = (rho < highest_density) & (this_border == True) if halo_selection.count(True) > 0: cluster_id.set_selected(halo_selection, -1) core_selection = (cluster_id == ic) & ~halo_selection highest_density = flex.max(rho.select(core_selection)) too_sparse = core_selection & ( rho.as_double() < highest_density / 10. ) # another heuristic if too_sparse.count(True) > 0: cluster_id.set_selected(too_sparse, -1) self.cluster_id_final = cluster_id.deep_copy() print("%d in the excluded halo" % ((cluster_id == -1).count(True)))
def generate_view_data (self) : from scitbx.array_family import flex from scitbx import graphics_utils settings = self.settings data_for_colors = data_for_radii = None data = self.data #self.work_array.data() if (isinstance(data, flex.double) and data.all_eq(0)): data = flex.double(data.size(), 1) if ((self.multiplicities is not None) and (settings.scale_colors_multiplicity)) : data_for_colors = self.multiplicities.data().as_double() assert data_for_colors.size() == data.size() elif (settings.sqrt_scale_colors) and (isinstance(data, flex.double)) : data_for_colors = flex.sqrt(data) else : data_for_colors = data.deep_copy() if ((self.multiplicities is not None) and (settings.scale_radii_multiplicity)) : #data_for_radii = data.deep_copy() data_for_radii = self.multiplicities.data().as_double() assert data_for_radii.size() == data.size() elif (settings.sqrt_scale_radii) and (isinstance(data, flex.double)) : data_for_radii = flex.sqrt(data) else : data_for_radii = data.deep_copy() if (settings.slice_mode) : data = data.select(self.slice_selection) if (not settings.keep_constant_scale) : data_for_radii = data_for_radii.select(self.slice_selection) data_for_colors = data_for_colors.select(self.slice_selection) if (settings.color_scheme in ["rainbow", "heatmap", "redblue"]) : colors = graphics_utils.color_by_property( properties=data_for_colors, selection=flex.bool(data_for_colors.size(), True), color_all=False, gradient_type=settings.color_scheme) elif (settings.color_scheme == "grayscale") : colors = graphics_utils.grayscale_by_property( properties=data_for_colors, selection=flex.bool(data_for_colors.size(), True), shade_all=False, invert=settings.black_background) else : if (settings.black_background) : base_color = (1.0,1.0,1.0) else : base_color = (0.0,0.0,0.0) colors = flex.vec3_double(data_for_colors.size(), base_color) if (settings.slice_mode) and (settings.keep_constant_scale) : colors = colors.select(self.slice_selection) data_for_radii = data_for_radii.select(self.slice_selection) uc = self.work_array.unit_cell() abc = uc.parameters()[0:3] min_dist = min(uc.reciprocal_space_vector((1,1,1))) min_radius = 0.20 * min_dist max_radius = 40 * min_dist if (settings.sqrt_scale_radii) and (not settings.scale_radii_multiplicity): data_for_radii = flex.sqrt(data_for_radii) if len(data_for_radii): max_value = flex.max(data_for_radii) scale = max_radius / max_value radii = data_for_radii * scale too_small = radii < min_radius if (too_small.count(True) > 0) : radii.set_selected(too_small, flex.double(radii.size(), min_radius)) assert radii.size() == colors.size() else: radii = flex.double() max_radius = 0 self.radii = radii self.max_radius = max_radius self.colors = colors
def run_detail(show_plot, save_plot): P = Profiler("0. Read data") import sys file_name = sys.argv[1] from xfel.clustering.singleframe import CellOnlyFrame from cctbx import crystal cells = [] for line in open(file_name, "r"): tokens = line.strip().split() unit_cell = tuple(float(x) for x in tokens[0:6]) space_group_symbol = tokens[6] crystal_symmetry = crystal.symmetry( unit_cell=unit_cell, space_group_symbol=space_group_symbol) cells.append(CellOnlyFrame(crystal_symmetry, path=None)) MM = [c.mm for c in cells] # get all metrical matrices MM_double = flex.double() for i in range(len(MM)): Tup = MM[i] for j in range(6): MM_double.append(Tup[j]) print(("There are %d cells" % (len(MM)))) coord_x = flex.double([c.uc[0] for c in cells]) coord_y = flex.double([c.uc[1] for c in cells]) if show_plot or save_plot: import matplotlib if not show_plot: # http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear matplotlib.use('Agg') # use a non-interactive backend from matplotlib import pyplot as plt plt.plot([c.uc[0] for c in cells], [c.uc[1] for c in cells], "k.", markersize=3.) plt.axes().set_aspect("equal") if save_plot: plt.savefig(plot_name, size_inches=(10, 10), dpi=300, bbox_inches='tight') if show_plot: plt.show() print("Now constructing a Dij matrix.") P = Profiler("1. compute Dij matrix") NN = len(MM) import omptbx omptbx.omp_set_num_threads(64) from cctbx.uctbx.determine_unit_cell import NCDist_matrix, NCDist_flatten #Dij = NCDist_matrix(MM_double) # double loop, less efficient evaluation of upper triangle Dij = NCDist_flatten(MM_double) # loop is flattened #from cctbx.uctbx.determine_unit_cell import NCDist # can this be refactored with MPI? #Dij = flex.double(flex.grid(NN,NN)) #for i in xrange(NN): # for j in xrange(i+1,NN): # Dij[i,j] = NCDist(MM[i], MM[j]) del P d_c = 10 # the distance cutoff, such that average item neighbors 1-2% of all items CM = clustering_manager(Dij=Dij, d_c=d_c) # Summarize the results here n_cluster = 1 + flex.max(CM.cluster_id_final) print(len(cells), "have been analyzed") print(("# ------------ %d CLUSTERS ----------------" % (n_cluster))) for i in range(n_cluster): item = flex.first_index(CM.cluster_id_maxima, i) print("Cluster %d. Central unit cell: item %d" % (i, item)) cells[item].crystal_symmetry.show_summary() print("Cluster has %d items, or %d after trimming borders" % ((CM.cluster_id_full == i).count(True), (CM.cluster_id_final == i).count(True))) print() appcolors = [ 'b', 'r', '#ff7f0e', '#2ca02c', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf' ] if show_plot: #Decision graph from matplotlib import pyplot as plt plt.plot(CM.rho, CM.delta, "r.", markersize=3.) for x in range(NN): if CM.cluster_id_maxima[x] >= 0: plt.plot([CM.rho[x]], [CM.delta[x]], "ro") plt.show() #No-halo plot from matplotlib import pyplot as plt colors = [appcolors[i % 10] for i in CM.cluster_id_full] plt.scatter(coord_x, coord_y, marker='o', color=colors, linewidths=0.4, edgecolor='k') for i in range(n_cluster): item = flex.first_index(CM.cluster_id_maxima, i) plt.plot([cells[item].uc[0]], [cells[item].uc[1]], 'y.') plt.axes().set_aspect("equal") plt.show() #Final plot halo = (CM.cluster_id_final == -1) core = ~halo plt.plot(coord_x.select(halo), coord_y.select(halo), "k.") colors = [appcolors[i % 10] for i in CM.cluster_id_final.select(core)] plt.scatter(coord_x.select(core), coord_y.select(core), marker="o", color=colors, linewidths=0.4, edgecolor='k') for i in range(n_cluster): item = flex.first_index(CM.cluster_id_maxima, i) plt.plot([cells[item].uc[0]], [cells[item].uc[1]], 'y.') plt.axes().set_aspect("equal") plt.show()
def plot_overall_completeness(completeness): completeness_range = range(-1, flex.max(completeness) + 1) completeness_counts = [completeness.count(n) for n in completeness_range] from matplotlib import pyplot as plt plt.plot(completeness_range, completeness_counts, "r+") plt.show()
def run(args, command_name="phenix.reflection_file_converter", simply_return_all_miller_arrays=False): command_line = (option_parser( usage="%s [options] reflection_file ..." % command_name, description="Example: %s w1.sca --mtz ." % command_name ).enable_symmetry_comprehensive().option( None, "--weak_symmetry", action="store_true", default=False, help="symmetry on command line is weaker than symmetry found in files" ).enable_resolutions().option( None, "--label", action="store", type="string", help="Substring of reflection data label or number", metavar="STRING" ).option( None, "--non_anomalous", action="store_true", default=False, help="Averages Bijvoet mates to obtain a non-anomalous array" ).option( None, "--r_free_label", action="store", type="string", help="Substring of reflection data label or number", metavar="STRING" ).option( None, "--r_free_test_flag_value", action="store", type="int", help="Value in R-free array indicating assignment to free set.", metavar="FLOAT" ).option( None, "--generate_r_free_flags", action="store_true", default=False, help="Generates a new array of random R-free flags" " (MTZ and CNS output only)." ).option( None, "--use_lattice_symmetry_in_r_free_flag_generation", dest="use_lattice_symmetry_in_r_free_flag_generation", action="store_true", default=True, help="group twin/pseudo symmetry related reflections together" " in r-free set (this is the default)." ).option( None, "--no_lattice_symmetry_in_r_free_flag_generation", dest="use_lattice_symmetry_in_r_free_flag_generation", action="store_false", help="opposite of --use-lattice-symmetry-in-r-free-flag-generation" ).option( None, "--r_free_flags_fraction", action="store", default=0.10, type="float", help="Target fraction free/work reflections (default: 0.10).", metavar="FLOAT" ).option( None, "--r_free_flags_max_free", action="store", default=2000, type="int", help="Maximum number of free reflections (default: 2000).", metavar="INT" ).option( None, "--r_free_flags_format", choices=( "cns", "ccp4", "shelx"), default="cns", help="Convention for generating R-free flags", metavar="cns|ccp4" ).option( None, "--output_r_free_label", action="store", type="string", help= "Label for newly generated R-free flags (defaults to R-free-flags)", default="R-free-flags", metavar="STRING" ).option( None, "--random_seed", action="store", type="int", help="Seed for random number generator (affects generation of" " R-free flags).", metavar="INT" ).option( None, "--change_of_basis", action="store", type="string", help="Change-of-basis operator: h,k,l or x,y,z" " or to_reference_setting, to_primitive_setting, to_niggli_cell," " to_inverse_hand", metavar="STRING" ).option( None, "--eliminate_invalid_indices", action="store_true", default=False, help="Remove indices which are invalid given the change of basis desired" ).option( None, "--expand_to_p1", action="store_true", default=False, help="Generates all symmetrically equivalent reflections." " The space group symmetry is reset to P1." " May be used in combination with --change_to_space_group to" " lower the symmetry." ).option( None, "--change_to_space_group", action="store", type="string", help="Changes the space group and merges equivalent reflections" " if necessary", metavar="SYMBOL|NUMBER" ).option( None, "--write_mtz_amplitudes", action="store_true", default=False, help="Converts intensities to amplitudes before writing MTZ format;" " requires --mtz_root_label" ).option( None, "--write_mtz_intensities", action="store_true", default=False, help="Converts amplitudes to intensities before writing MTZ format;" " requires --mtz_root_label" ).option( None, "--remove_negatives", action="store_true", default=False, help="Remove negative intensities or amplitudes from the data set" ).option( None, "--massage_intensities", action="store_true", default=False, help="'Treat' negative intensities to get a positive amplitude." " |Fnew| = sqrt((Io+sqrt(Io**2 +2sigma**2))/2.0). Requires" " intensities as input and the flags --mtz," " --write_mtz_amplitudes and --mtz_root_label." ).option( None, "--scale_max", action="store", type="float", help="Scales data such that the maximum is equal to the given value", metavar="FLOAT" ).option( None, "--scale_factor", action="store", type="float", help="Multiplies data with the given factor", metavar="FLOAT" ).option( None, "--write_unmerged", action="store_true", default=False, help="Do not perform any merging of input data" ).option( None, "--sca", action="store", type="string", help= "write data to Scalepack FILE ('--sca .' copies name of input file)", metavar="FILE" ).option( None, "--mtz", action="store", type="string", help="write data to MTZ FILE ('--mtz .' copies name of input file)", metavar="FILE" ).option( None, "--mtz_root_label", action="store", type="string", help="Root label for MTZ file (e.g. Fobs)", metavar="STRING" ).option( None, "--cns", action="store", type="string", help="write data to CNS FILE ('--cns .' copies name of input file)", metavar="FILE" ).option( None, "--shelx", action="store", type="string", help= "write intensity or amplitude data to SHELX FILE ('--shelx .' copies name of input file)", metavar="FILE")).process(args=args) co = command_line.options if (co.random_seed is not None): random.seed(co.random_seed) flex.set_random_seed(value=co.random_seed) if (co.write_mtz_amplitudes and co.write_mtz_intensities): print() print("--write_mtz_amplitudes and --write_mtz_intensities" \ " are mutually exclusive.") print() return None if (co.write_mtz_amplitudes or co.write_mtz_intensities): if (co.mtz_root_label is None): print() print("--write_mtz_amplitudes and --write_mtz_intensities" \ " require --mtz_root_label.") print() return None if (co.scale_max is not None and co.scale_factor is not None): print() print("--scale_max and --scale_factor are mutually exclusive.") print() return None if (len(command_line.args) == 0): command_line.parser.show_help() return None all_miller_arrays = reflection_file_reader.collect_arrays( file_names=command_line.args, crystal_symmetry=None, force_symmetry=False, merge_equivalents=False, discard_arrays=False, verbose=1) if (simply_return_all_miller_arrays): return all_miller_arrays if (len(all_miller_arrays) == 0): print() print("No reflection data found in input file%s." % (plural_s(len(command_line.args))[1])) print() return None label_table = reflection_file_utils.label_table( miller_arrays=all_miller_arrays) selected_array = label_table.select_array(label=co.label, command_line_switch="--label") if (selected_array is None): return None r_free_flags = None r_free_info = None if (co.r_free_label is not None): r_free_flags = label_table.match_data_label( label=co.r_free_label, command_line_switch="--r_free_label") if (r_free_flags is None): return None r_free_info = str(r_free_flags.info()) if (not r_free_flags.is_bool_array()): test_flag_value = reflection_file_utils.get_r_free_flags_scores( miller_arrays=[r_free_flags], test_flag_value=co.r_free_test_flag_value).test_flag_values[0] if (test_flag_value is None): if (co.r_free_test_flag_value is None): raise Sorry( "Cannot automatically determine r_free_test_flag_value." " Please use --r_free_test_flag_value to specify a value." ) else: raise Sorry("Invalid --r_free_test_flag_value.") r_free_flags = r_free_flags.customized_copy( data=(r_free_flags.data() == test_flag_value)) print("Selected data:") print(" ", selected_array.info()) print(" Observation type:", selected_array.observation_type()) print() if (r_free_info is not None): print("R-free flags:") print(" ", r_free_info) print() processed_array = selected_array.customized_copy( crystal_symmetry=selected_array.join_symmetry( other_symmetry=command_line.symmetry, force=not co.weak_symmetry)).set_observation_type( selected_array.observation_type()) if (r_free_flags is not None): r_free_flags = r_free_flags.customized_copy( crystal_symmetry=processed_array) print("Input crystal symmetry:") crystal.symmetry.show_summary(processed_array, prefix=" ") print() if (processed_array.unit_cell() is None): command_line.parser.show_help() print( "Unit cell parameters unknown. Please use --symmetry or --unit_cell." ) print() return None if (processed_array.space_group_info() is None): command_line.parser.show_help() print("Space group unknown. Please use --symmetry or --space_group.") print() return None if (r_free_flags is not None): r_free_flags = r_free_flags.customized_copy( crystal_symmetry=processed_array) if (co.change_of_basis is not None): processed_array, cb_op = processed_array.apply_change_of_basis( change_of_basis=co.change_of_basis, eliminate_invalid_indices=co.eliminate_invalid_indices) if (r_free_flags is not None): r_free_flags = r_free_flags.change_basis(cb_op=cb_op) if (not processed_array.is_unique_set_under_symmetry() and not co.write_unmerged): print("Merging symmetry-equivalent values:") merged = processed_array.merge_equivalents() merged.show_summary(prefix=" ") print() processed_array = merged.array() del merged processed_array.show_comprehensive_summary(prefix=" ") print() if (r_free_flags is not None and not r_free_flags.is_unique_set_under_symmetry() and not co.write_unmerged): print("Merging symmetry-equivalent R-free flags:") merged = r_free_flags.merge_equivalents() merged.show_summary(prefix=" ") print() r_free_flags = merged.array() del merged r_free_flags.show_comprehensive_summary(prefix=" ") print() if (co.expand_to_p1): print("Expanding symmetry and resetting space group to P1:") if (r_free_flags is not None): raise Sorry( "--expand_to_p1 not supported for arrays of R-free flags.") processed_array = processed_array.expand_to_p1() processed_array.show_comprehensive_summary(prefix=" ") print() if (co.change_to_space_group is not None): if (r_free_flags is not None): raise Sorry( "--change_to_space_group not supported for arrays of R-free flags." ) new_space_group_info = sgtbx.space_group_info( symbol=co.change_to_space_group) print("Change to space group:", new_space_group_info) new_crystal_symmetry = crystal.symmetry( unit_cell=processed_array.unit_cell(), space_group_info=new_space_group_info, assert_is_compatible_unit_cell=False) if (not new_crystal_symmetry.unit_cell().is_similar_to( processed_array.unit_cell())): print(" *************") print(" W A R N I N G") print(" *************") print( " Unit cell parameters adapted to new space group symmetry are" ) print(" significantly different from input unit cell parameters:") print(" Input unit cell parameters:", \ processed_array.unit_cell()) print(" Adapted unit cell parameters:", \ new_crystal_symmetry.unit_cell()) processed_array = processed_array.customized_copy( crystal_symmetry=new_crystal_symmetry) print() if (not processed_array.is_unique_set_under_symmetry() and not co.write_unmerged): print(" Merging values symmetry-equivalent under new symmetry:") merged = processed_array.merge_equivalents() merged.show_summary(prefix=" ") print() processed_array = merged.array() del merged processed_array.show_comprehensive_summary(prefix=" ") print() if (processed_array.anomalous_flag() and co.non_anomalous): print("Converting data array from anomalous to non-anomalous.") if (not processed_array.is_xray_intensity_array()): processed_array = processed_array.average_bijvoet_mates() else: processed_array = processed_array.average_bijvoet_mates() processed_array.set_observation_type_xray_intensity() if (r_free_flags is not None and r_free_flags.anomalous_flag() and co.non_anomalous): print("Converting R-free flags from anomalous to non-anomalous.") r_free_flags = r_free_flags.average_bijvoet_mates() d_max = co.low_resolution d_min = co.resolution if (d_max is not None or d_min is not None): if (d_max is not None): print("Applying low resolution cutoff: d_max=%.6g" % d_max) if (d_min is not None): print("Applying high resolution cutoff: d_min=%.6g" % d_min) processed_array = processed_array.resolution_filter(d_max=d_max, d_min=d_min) print("Number of reflections:", processed_array.indices().size()) print() if (co.scale_max is not None): print("Scaling data such that the maximum value is: %.6g" % co.scale_max) processed_array = processed_array.apply_scaling( target_max=co.scale_max) print() if (co.scale_factor is not None): print("Multiplying data with the factor: %.6g" % co.scale_factor) processed_array = processed_array.apply_scaling(factor=co.scale_factor) print() if (([co.remove_negatives, co.massage_intensities]).count(True) == 2): raise Sorry("It is not possible to use --remove_negatives and" " --massage_intensities at the same time.") if (co.remove_negatives): if processed_array.is_real_array(): print("Removing negatives items") processed_array = processed_array.select( processed_array.data() > 0) if processed_array.sigmas() is not None: processed_array = processed_array.select( processed_array.sigmas() > 0) else: raise Sorry( "--remove_negatives not applicable to complex data arrays.") if (co.massage_intensities): if processed_array.is_real_array(): if processed_array.is_xray_intensity_array(): if (co.mtz is not None): if (co.write_mtz_amplitudes): print( "The supplied intensities will be used to estimate" ) print(" amplitudes in the following way: ") print( " Fobs = Sqrt[ (Iobs + Sqrt(Iobs**2 + 2sigmaIobs**2))/2 ]" ) print(" Sigmas are estimated in a similar manner.") print() processed_array = processed_array.enforce_positive_amplitudes( ) else: raise Sorry( "--write_mtz_amplitudes has to be specified when using" " --massage_intensities") else: raise Sorry( "--mtz has to be used when using --massage_intensities" ) else: raise Sorry( "Intensities must be supplied when using the option" " --massage_intensities") else: raise Sorry( "--massage_intensities not applicable to complex data arrays.") if (not co.generate_r_free_flags): if (r_free_flags is None): r_free_info = [] else: if (r_free_flags.anomalous_flag() != processed_array.anomalous_flag()): if (processed_array.anomalous_flag()): is_not = ("", " not") else: is_not = (" not", "") raise Sorry( "The data array is%s anomalous but the R-free array is%s.\n" % is_not + " Please try --non_anomalous.") r_free_info = ["R-free flags source: " + r_free_info] if (not r_free_flags.indices().all_eq(processed_array.indices())): processed_array = processed_array.map_to_asu() r_free_flags = r_free_flags.map_to_asu().common_set( processed_array) n_missing_r_free_flags = processed_array.indices().size() \ - r_free_flags.indices().size() if (n_missing_r_free_flags != 0): raise Sorry( "R-free flags not compatible with data array:" " missing flag for %d reflections selected for output." % n_missing_r_free_flags) else: if (r_free_flags is not None): raise Sorry( "--r_free_label and --generate_r_free_flags are mutually exclusive." ) print("Generating a new array of R-free flags:") r_free_flags = processed_array.generate_r_free_flags( fraction=co.r_free_flags_fraction, max_free=co.r_free_flags_max_free, use_lattice_symmetry=co. use_lattice_symmetry_in_r_free_flag_generation, format=co.r_free_flags_format) test_flag_value = True if (co.r_free_flags_format == "ccp4"): test_flag_value = 0 elif (co.r_free_flags_format == "shelx"): test_flag_value = -1 r_free_as_bool = r_free_flags.customized_copy( data=r_free_flags.data() == test_flag_value) r_free_info = ["R-free flags generated by %s:" % command_name] r_free_info.append(" " + date_and_time()) r_free_info.append(" fraction: %.6g" % co.r_free_flags_fraction) r_free_info.append(" max_free: %s" % str(co.r_free_flags_max_free)) r_free_info.append(" size of work set: %d" % r_free_as_bool.data().count(False)) r_free_info.append(" size of free set: %d" % r_free_as_bool.data().count(True)) r_free_info_str = StringIO() r_free_as_bool.show_r_free_flags_info(prefix=" ", out=r_free_info_str) if (co.r_free_flags_format == "ccp4"): r_free_info.append(" convention: CCP4 (test=0, work=1-%d)" % flex.max(r_free_flags.data())) elif (co.r_free_flags_format == "shelx"): r_free_info.append(" convention: SHELXL (test=-1, work=1)") else: r_free_info.append(" convention: CNS/X-PLOR (test=1, work=0)") print("\n".join(r_free_info[2:4])) print(r_free_info[-1]) print(r_free_info_str.getvalue()) print() n_output_files = 0 if (co.sca is not None): if (co.generate_r_free_flags): raise Sorry("Cannot write R-free flags to Scalepack file.") file_name = reflection_file_utils.construct_output_file_name( input_file_names=[selected_array.info().source], user_file_name=co.sca, file_type_label="Scalepack", file_extension="sca") print("Writing Scalepack file:", file_name) iotbx.scalepack.merge.write(file_name=file_name, miller_array=processed_array) n_output_files += 1 print() if (co.mtz is not None): file_name = reflection_file_utils.construct_output_file_name( input_file_names=[selected_array.info().source], user_file_name=co.mtz, file_type_label="MTZ", file_extension="mtz") print("Writing MTZ file:", file_name) mtz_history_buffer = flex.std_string() mtz_history_buffer.append(date_and_time()) mtz_history_buffer.append("> program: %s" % command_name) mtz_history_buffer.append( "> input file name: %s" % os.path.basename(selected_array.info().source)) mtz_history_buffer.append( "> input directory: %s" % os.path.dirname(os.path.abspath(selected_array.info().source))) mtz_history_buffer.append("> input labels: %s" % selected_array.info().label_string()) mtz_output_array = processed_array if (co.write_mtz_amplitudes): if (not mtz_output_array.is_xray_amplitude_array()): print(" Converting intensities to amplitudes.") mtz_output_array = mtz_output_array.f_sq_as_f() mtz_history_buffer.append( "> Intensities converted to amplitudes.") elif (co.write_mtz_intensities): if (not mtz_output_array.is_xray_intensity_array()): print(" Converting amplitudes to intensities.") mtz_output_array = mtz_output_array.f_as_f_sq() mtz_history_buffer.append( "> Amplitudes converted to intensities.") column_root_label = co.mtz_root_label if (column_root_label is None): # XXX 2013-03-29: preserve original root label by default # XXX 2014-12-16: skip trailing "(+)" in root_label if anomalous column_root_label = selected_array.info().labels[0] column_root_label = remove_anomalous_suffix_if_necessary( miller_array=selected_array, column_root_label=column_root_label) mtz_dataset = mtz_output_array.as_mtz_dataset( column_root_label=column_root_label) del mtz_output_array if (r_free_flags is not None): mtz_dataset.add_miller_array( miller_array=r_free_flags, column_root_label=co.output_r_free_label) for line in r_free_info: mtz_history_buffer.append("> " + line) mtz_history_buffer.append("> output file name: %s" % os.path.basename(file_name)) mtz_history_buffer.append("> output directory: %s" % os.path.dirname(os.path.abspath(file_name))) mtz_object = mtz_dataset.mtz_object() mtz_object.add_history(mtz_history_buffer) mtz_object.write(file_name=file_name) n_output_files += 1 print() if (co.cns is not None): file_name = reflection_file_utils.construct_output_file_name( input_file_names=[selected_array.info().source], user_file_name=co.cns, file_type_label="CNS", file_extension="cns") print("Writing CNS file:", file_name) processed_array.export_as_cns_hkl( file_object=open(file_name, "w"), file_name=file_name, info=["source of data: " + str(selected_array.info())] + r_free_info, r_free_flags=r_free_flags) n_output_files += 1 print() if (co.shelx is not None): if (co.generate_r_free_flags): raise Sorry("Cannot write R-free flags to SHELX file.") file_extension = "hkl" if (co.shelx.endswith("shelx")): file_extension = "shelx" file_name = reflection_file_utils.construct_output_file_name( input_file_names=[selected_array.info().source], user_file_name=co.shelx, file_type_label="SHELX", file_extension=file_extension) if processed_array.is_xray_intensity_array(): data_status = "HKLF 4 (intensity)" else: data_status = "HKLF 3 (amplitude)" print("Writing SHELX", data_status, "file:", file_name) processed_array.export_as_shelx_hklf(open(file_name, "w"), full_dynamic_range=True) n_output_files += 1 print() if (n_output_files == 0): command_line.parser.show_help() print("Please specify at least one output file format,", end=' ') print("e.g. --mtz, --sca, etc.") print() return None return processed_array
def plot_projections( projections, filename=None, colours=None, marker_size=3, font_size=6, gridsize=None, label_indices=False, epochs=None, colour_map=None, ): projections_all = projections # http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear matplotlib.use("Agg") # use a non-interactive backend from matplotlib import pylab, pyplot if epochs is not None and colour_map is not None: epochs = flex.double(epochs) epochs -= flex.min(epochs) epochs /= flex.max(epochs) cmap = matplotlib.cm.get_cmap(colour_map) colours = [cmap(e) for e in epochs] elif colours is None or len(colours) == 0: colours = ["b"] * len(projections_all) elif len(colours) < len(projections_all): colours = colours * len(projections_all) fig = pyplot.figure() pyplot.scatter([0], [0], marker="+", c="0.75", s=100) cir = pylab.Circle((0, 0), radius=1.0, fill=False, color="0.75") pylab.gca().add_patch(cir) if gridsize is not None: x = flex.double() y = flex.double() for i, projections in enumerate(projections_all): x_, y_ = projections.parts() x.extend(x_) y.extend(y_) hb = pyplot.hexbin(x, y, gridsize=gridsize, linewidths=0.2) pyplot.colorbar(hb) else: for i, projections in enumerate(projections_all): x, y = projections.parts() pyplot.scatter( x.as_numpy_array(), y.as_numpy_array(), c=colours[i], s=marker_size, edgecolors="none", ) if label_indices: for j, (hkl, proj) in enumerate(zip(label_indices, projections)): # hack to not write two labels on top of each other p1, p2 = (projections - proj).parts() if (flex.sqrt(flex.pow2(p1) + flex.pow2(p2)) < 1e-3).iselection()[0] != j: continue pyplot.text(proj[0], proj[1], str(hkl), fontsize=font_size) fig.axes[0].set_aspect("equal") pyplot.xlim(-1.1, 1.1) pyplot.ylim(-1.1, 1.1) if filename is not None: pyplot.savefig(filename, dpi=300)
def refine_rotx_roty2(OO,enable_rotational_target=True): helper = OO.per_frame_helper_factory() helper.restart() if enable_rotational_target: print "Trying least squares minimization of excursions", from scitbx.lstbx import normal_eqns_solving iterations = normal_eqns_solving.naive_iterations( non_linear_ls = helper, gradient_threshold = 1.E-10) results = helper.x print "with %d reflections"%len(OO.parent.indexed_pairs), print "result %6.2f degrees"%(results[1]*180./math.pi), print "result %6.2f degrees"%(results[0]*180./math.pi) if False: # Excursion histogram print "The input mosaicity is %7.3f deg full width"%OO.parent.inputai.getMosaicity() # final histogram if OO.pvr_fix: final = 360.* helper.fvec_callable_pvr(results) else: final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results) rmsdexc = math.sqrt(flex.mean(final*final)) from matplotlib import pyplot as plt nbins = len(final)//20 n,bins,patches = plt.hist(final, nbins, normed=0, facecolor="orange", alpha=0.75) plt.xlabel("Rotation on e1 axis, rmsd %7.3f deg"%rmsdexc) plt.title("Histogram of cctbx.xfel misorientation") plt.axis([-0.5,0.5,0,100]) plt.plot([rmsdexc],[18],"b|") plt.show() # Determine optimal mosaicity and domain size model (monochromatic) if OO.pvr_fix: final = 360.* helper.fvec_callable_pvr(results) else: final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results) #Guard against misindexing -- seen in simulated data, with zone nearly perfectly aligned guard_stats = flex.max(final), flex.min(final) if False and REMOVETEST_KILLING_LEGITIMATE_EXCURSIONS (guard_stats[0] > 2.0 or guard_stats[1] < -2.0): raise Exception("Misindexing diagnosed by meaningless excursion angle (bandpass_gaussian model)"); print "The mean excursion is %7.3f degrees"%(flex.mean(final)) two_thetas = helper.last_set_orientation.unit_cell().two_theta(OO.reserve_indices,OO.central_wavelength_ang,deg=True) dspacings = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) dspace_sq = dspacings * dspacings excursion_rad = final * math.pi/ 180. # First -- try to get a reasonable envelope for the observed excursions. ## minimum of three regions; maximum of 50 measurements in each bin print "fitting parameters on %d spots"%len(excursion_rad) n_bins = min(max(3, len(excursion_rad)//25),50) bin_sz = len(excursion_rad)//n_bins print "nbins",n_bins,"bin_sz",bin_sz order = flex.sort_permutation(two_thetas) two_thetas_env = flex.double() dspacings_env = flex.double() excursion_rads_env = flex.double() for x in xrange(0,n_bins): subset = order[x*bin_sz:(x+1)*bin_sz] two_thetas_env.append( flex.mean(two_thetas.select(subset)) ) dspacings_env.append( flex.mean(dspacings.select(subset))) excursion_rads_env.append( flex.max( flex.abs( excursion_rad.select(subset)))) # Second -- parameter fit ## solve the normal equations sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env) sum_inv_u = flex.sum(dspacings_env) sum_te_u = flex.sum(dspacings_env * excursion_rads_env) sum_te = flex.sum(excursion_rads_env) Normal_Mat = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env))) Vector = col((sum_te_u, sum_te)) solution = Normal_Mat.inverse() * Vector s_ang = 1./(2*solution[0]) print "Best LSQ fit Scheerer domain size is %9.2f ang"%( s_ang) tan_phi_rad = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. * s_ang) tan_phi_deg = tan_phi_rad * 180./math.pi k_degrees = solution[1]* 180./math.pi print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees) tan_outer_deg = tan_phi_deg + k_degrees if OO.mosaic_refinement_target=="ML": from xfel.mono_simulation.max_like import minimizer print "input", s_ang,2. * solution[1]*180/math.pi # coerce the estimates to be positive for max-likelihood lower_limit_domain_size = math.pow( helper.last_set_orientation.unit_cell().volume(), 1./3.)*20 # 10-unit cell block size minimum reasonable domain d_estimate = max(s_ang, lower_limit_domain_size) M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]), Deff = d_estimate) print "output",1./M.x[0], M.x[1]*180./math.pi tan_phi_rad_ML = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. / M.x[0]) tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi # bugfix: Need factor of 0.5 because the plot shows half mosaicity (displacement from the center point defined as zero) tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi if OO.parent.horizons_phil.integration.mosaic.enable_polychromatic: # add code here to perform polychromatic modeling. """ get miller indices DONE get model-predicted mono-wavelength centroid S1 vectors back-convert S1vec, with mono-wavelength, to detector-plane position, factoring in subpixel correction compare with spot centroid measured position compare with locus of bodypixels """ print list(OO.reserve_indices) print len(OO.reserve_indices), len(two_thetas) positions = [ OO.ucbp3.simple_forward_calculation_spot_position( wavelength = OO.central_wavelength_ang, observation_no = obsno).position for obsno in xrange(len(OO.parent.indexed_pairs))] print len(positions) print positions # model-predicted positions print len(OO.parent.spots) print OO.parent.indexed_pairs print OO.parent.spots print len(OO.parent.spots) meas_spots = [OO.parent.spots[pair["spot"]] for pair in OO.parent.indexed_pairs] # for xspot in meas_spots: # xspot.ctr_mass_x(),xspot.ctr_mass_y() # xspot.max_pxl_x() # xspot.bodypixels # xspot.ctr_mass_x() # Do some work to calculate an rmsd diff_vecs = flex.vec3_double() for p,xspot in zip(positions, meas_spots): diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0)) # could use diff_vecs.rms_length() diff_vecs_sq = diff_vecs.dot(diff_vecs) mean_diff_vec_sq = flex.mean(diff_vecs_sq) rmsd = math.sqrt(mean_diff_vec_sq) print "mean obs-pred diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd) positions_to_fictitious = [ OO.ucbp3.simple_forward_calculation_spot_position( wavelength = OO.central_wavelength_ang, observation_no = obsno).position_to_fictitious for obsno in xrange(len(OO.parent.indexed_pairs))] # Do some work to calculate an rmsd diff_vecs = flex.vec3_double() for p,xspot in zip(positions_to_fictitious, meas_spots): diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0)) rmsd = diff_vecs.rms_length() print "mean obs-pred_to_fictitious diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd) """ actually, it might be better if the entire set of experimental observations is transformed into the ideal detector plane, for the purposes of poly_treatment. start here. Now it would be good to actually implement probability of observing a body pixel given the model. We have everything needed right here. """ if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Image plot: obs and predicted positions + bodypixels from matplotlib import pyplot as plt plt.plot( [p[0] for p in positions_to_fictitious], [p[1] for p in positions_to_fictitious], "r.") plt.plot( [xspot.ctr_mass_y() for xspot in meas_spots], [xspot.ctr_mass_x() for xspot in meas_spots], "g.") bodypx = [] for xspot in meas_spots: for body in xspot.bodypixels: bodypx.append(body) plt.plot( [b.y for b in bodypx], [b.x for b in bodypx], "b.") plt.axes().set_aspect("equal") plt.show() print "MEAN excursion",flex.mean(final), if OO.mosaic_refinement_target=="ML": print "mosaicity deg FW=",M.x[1]*180./math.pi else: print if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Excursion vs resolution fit AD1TF7B_MAX2T = 30. AD1TF7B_MAXDP = 1. from matplotlib import pyplot as plt fig = plt.figure() plt.plot(two_thetas, final, "bo") mean = flex.mean(final) minplot = flex.min(two_thetas) plt.plot([0,minplot],[mean,mean],"k-") LR = flex.linear_regression(two_thetas, final) #LR.show_summary() model_y = LR.slope()*two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") print helper.last_set_orientation.unit_cell() #for sdp,tw in zip (dspacings,two_thetas): #print sdp,tw if OO.mosaic_refinement_target=="ML": plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas))) plt.plot(two_thetas, tan_phi_deg_ML, "r.") plt.plot(two_thetas, -tan_phi_deg_ML, "r.") plt.plot(two_thetas, tan_outer_deg_ML, "g.") plt.plot(two_thetas, -tan_outer_deg_ML, "g.") else: plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r|") plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r|") plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r-") plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r-") plt.plot(two_thetas, tan_phi_deg, "r.") plt.plot(two_thetas, -tan_phi_deg, "r.") plt.plot(two_thetas, tan_outer_deg, "g.") plt.plot(two_thetas, -tan_outer_deg, "g.") plt.xlim([0,AD1TF7B_MAX2T]) plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP]) OO.parent.show_figure(plt,fig,"psi") plt.close() from xfel.mono_simulation.util import green_curve_area,ewald_proximal_volume if OO.mosaic_refinement_target=="ML": OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML) OO.parent.inputai.setMosaicity(M.x[1]*180./math.pi) # full width, degrees OO.parent.ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi) OO.parent.ML_domain_size_ang = 1./M.x[0] OO.parent.ewald_proximal_volume = ewald_proximal_volume( wavelength_ang = OO.central_wavelength_ang, resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff, domain_size_ang = 1./M.x[0], full_mosaicity_rad = M.x[1]) return results, helper.last_set_orientation,1./M.x[0] # full width domain size, angstroms else: assert OO.mosaic_refinement_target=="LSQ" OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg) OO.parent.inputai.setMosaicity(2*k_degrees) # full width OO.parent.ML_half_mosaicity_deg = k_degrees OO.parent.ML_domain_size_ang = s_ang OO.parent.ewald_proximal_volume = ewald_proximal_volume( wavelength_ang = OO.central_wavelength_ang, resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff, domain_size_ang = s_ang, full_mosaicity_rad = 2*k_degrees*math.pi/180.) return results, helper.last_set_orientation,s_ang # full width domain size, angstroms
def exercise_2(): for use_reference in [True, False, None]: pdb_inp = iotbx.pdb.input(lines=flex.std_string( pdb_str_2.splitlines()), source_info=None) model = manager(model_input=pdb_inp, log=null_out()) grm = model.get_restraints_manager().geometry xrs2 = model.get_xray_structure() awl2 = model.get_hierarchy().atoms_with_labels() pdb_inp3 = iotbx.pdb.input(source_info=None, lines=pdb_str_3) xrs3 = pdb_inp3.xray_structure_simple() ph3 = pdb_inp3.construct_hierarchy() ph3.atoms().reset_i_seq() awl3 = ph3.atoms_with_labels() sites_cart_reference = flex.vec3_double() selection = flex.size_t() reference_names = ["CG", "CD", "NE", "CZ", "NH1", "NH2"] for a2, a3 in zip(tuple(awl2), tuple(awl3)): assert a2.resname == a3.resname assert a2.name == a3.name assert a2.i_seq == a3.i_seq if (a2.resname == "ARG" and a2.name.strip() in reference_names): selection.append(a2.i_seq) sites_cart_reference.append(a3.xyz) assert selection.size() == len(reference_names) selection_bool = flex.bool(xrs2.scatterers().size(), selection) if (use_reference): grm.adopt_reference_coordinate_restraints_in_place( reference.add_coordinate_restraints( sites_cart=sites_cart_reference, selection=selection, sigma=0.01)) elif (use_reference is None): grm.adopt_reference_coordinate_restraints_in_place( reference.add_coordinate_restraints( sites_cart=sites_cart_reference, selection=selection, sigma=0.01)) grm.remove_reference_coordinate_restraints_in_place( selection=selection) d1 = flex.mean( flex.sqrt((xrs2.sites_cart().select(selection) - xrs3.sites_cart().select(selection)).dot())) print "distance start (use_reference: %s): %6.4f" % ( str(use_reference), d1) assert d1 > 4.0 assert approx_equal( flex.max( flex.sqrt((xrs2.sites_cart().select(~selection_bool) - xrs3.sites_cart().select(~selection_bool)).dot())), 0) from cctbx import geometry_restraints import mmtbx.refinement.geometry_minimization import scitbx.lbfgs grf = geometry_restraints.flags.flags(default=True) sites_cart = xrs2.sites_cart() minimized = mmtbx.refinement.geometry_minimization.lbfgs( sites_cart=sites_cart, correct_special_position_tolerance=1.0, geometry_restraints_manager=grm, sites_cart_selection=flex.bool(sites_cart.size(), selection), geometry_restraints_flags=grf, lbfgs_termination_params=scitbx.lbfgs.termination_parameters( max_iterations=5000)) xrs2.set_sites_cart(sites_cart=sites_cart) d2 = flex.mean( flex.sqrt((xrs2.sites_cart().select(selection) - xrs3.sites_cart().select(selection)).dot())) print "distance final (use_reference: %s): %6.4f" % ( str(use_reference), d2) if (use_reference): assert d2 < 0.005, "failed: %f<0.05" % d2 else: assert d2 > 4.0, d2 assert approx_equal( flex.max( flex.sqrt((xrs2.sites_cart().select(~selection_bool) - xrs3.sites_cart().select(~selection_bool)).dot())), 0)
def plot_result(metric, result): if metric == metrics.CC_HALF: return plots.cc_half_plot( result.d_star_sq, result.y_obs, cc_half_critical_values=result.critical_values, cc_half_fit=result.y_fit, d_min=result.d_min, ) else: d = { metrics.MISIGMA: "Merged <I/σ(I)>", metrics.ISIGMA: "Unmerged <I/σ(I)>", metrics.I_MEAN_OVER_SIGMA_MEAN: "<I>/<σ(I)>", metrics.RMERGE: "R<sub>merge</sub> ", metrics.COMPLETENESS: "Completeness", } d_star_sq_tickvals, d_star_sq_ticktext = plots.d_star_sq_to_d_ticks( result.d_star_sq, 5) return { "data": [ { "x": list(result.d_star_sq), # d_star_sq "y": list(result.y_obs), "type": "scatter", "name": "y_obs", }, ({ "x": list(result.d_star_sq), "y": list(result.y_fit), "type": "scatter", "name": "y_fit", "line": { "color": "rgb(47, 79, 79)" }, } if result.y_fit else {}), ({ "x": [uctbx.d_as_d_star_sq(result.d_min)] * 2, "y": [ 0, max( 1, flex.max(result.y_obs), flex.max(result.y_fit) if result.y_fit else 0, ), ], "type": "scatter", "name": f"d_min = {result.d_min:.2f} Å", "mode": "lines", "line": { "color": "rgb(169, 169, 169)", "dash": "dot" }, } if result.d_min else {}), ], "layout": { "title": f"{d.get(metric)} vs. resolution", "xaxis": { "title": "Resolution (Å)", "tickvals": d_star_sq_tickvals, "ticktext": d_star_sq_ticktext, }, "yaxis": { "title": d.get(metric), "rangemode": "tozero" }, }, }
def __init__(self, fmodels, model, max_number_of_iterations=25, number_of_macro_cycles=3, occupancy_max=None, occupancy_min=None, log=None, exclude_hd=False): self.show(fmodels=fmodels, log=log, message="occupancy refinement: start") fmodels.update_xray_structure( xray_structure=model.get_xray_structure(), update_f_calc=True) selections = model.refinement_flags.s_occupancies # exclude H or D from refinement if requested if (exclude_hd): hd_sel = model.get_hd_selection() tmp_sel = [] for sel in selections: tmp_sel_ = [] for sel_ in sel: tmp_sel__ = flex.size_t() for sel__ in sel_: if (not hd_sel[sel__]): tmp_sel__.append(sel__) if (tmp_sel__.size() > 0): tmp_sel_.append(tmp_sel__) if (len(tmp_sel_) > 0): tmp_sel.append(tmp_sel_) selections = tmp_sel # if (len(selections) > 0): i_selection = flex.size_t() for s in selections: for ss in s: i_selection.extend(ss) fmodels.fmodel_xray().xray_structure.scatterers().flags_set_grads( state=False) fmodels.fmodel_xray().xray_structure.scatterers( ).flags_set_grad_occupancy(iselection=i_selection) fmodels.fmodel_xray().xray_structure.adjust_occupancy( occ_max=occupancy_max, occ_min=occupancy_min, selection=i_selection) xray_structure_dc = fmodels.fmodel_xray().xray_structure.\ deep_copy_scatterers() par_initial = flex.double() occupancies = xray_structure_dc.scatterers().extract_occupancies() constrained_groups_selections = [] group_counter = 0 for sel in selections: ss = [] for sel_ in sel: ss.append(group_counter) group_counter += 1 val = flex.mean(occupancies.select(sel_)) par_initial.append(val) constrained_groups_selections.append(ss) minimized = None for macro_cycle in xrange(number_of_macro_cycles): if (minimized is not None): par_initial = minimized.par_min minimized = minimizer( fmodels=fmodels, selections=selections, constrained_groups_selections=constrained_groups_selections, par_initial=par_initial, max_number_of_iterations=max_number_of_iterations) if (minimized is not None): par_initial = minimized.par_min set_refinable_parameters( xray_structure=fmodels.fmodel_xray().xray_structure, parameters=par_initial, selections=selections, enforce_positivity=(occupancy_min >= 0)) fmodels.fmodel_xray().xray_structure.adjust_occupancy( occ_max=occupancy_max, occ_min=occupancy_min, selection=i_selection) xray_structure_final = fmodels.fmodel_xray().xray_structure model.set_xray_structure(xray_structure_final) fmodels.update_xray_structure(xray_structure=xray_structure_final, update_f_calc=True) refined_occ = xray_structure_final.scatterers().extract_occupancies().\ select(i_selection) assert flex.min(refined_occ) >= occupancy_min assert flex.max(refined_occ) <= occupancy_max self.show(fmodels=fmodels, log=log, message="occupancy refinement: end")
def test_1(): tstart = time.time() fraction_missing = 0.1 d_min = 1.5 # create dummy model symmetry = crystal.symmetry(unit_cell=(15.67, 25.37, 35.68, 90, 90, 90), space_group_symbol="P 21 21 21") structure = xray.structure(crystal_symmetry=symmetry) for k in xrange(1000): scatterer = xray.scatterer( site=((1. + k * abs(math.sin(k))) / 1000.0, (1. + k * abs(math.cos(k))) / 1000.0, (1. + k) / 1000.0), u=abs(math.cos(k)) * 100. / (8. * math.pi**2), occupancy=1.0, scattering_type="C") structure.add_scatterer(scatterer) # partial model n_keep = int(round(structure.scatterers().size() * (1 - fraction_missing))) partial_structure = xray.structure(special_position_settings=structure) partial_structure.add_scatterers(structure.scatterers()[:n_keep]) # fcalc (partial model), fobs (fcalc full model) f_calc = structure.structure_factors(d_min=d_min, anomalous_flag=False).f_calc() f_calc_partial = partial_structure.structure_factors( d_min=d_min, anomalous_flag=False).f_calc() f_obs = abs(f_calc) f_calc = abs(f_calc_partial) d_star_sq = 1. / flex.pow2(f_obs.d_spacings().data()) assert approx_equal(flex.max(f_calc.data()), 6810.19834824) assert approx_equal(flex.min(f_calc.data()), 0.019589341727) assert approx_equal(flex.mean(f_calc.data()), 76.651506629) assert approx_equal(flex.max(f_obs.data()), 6962.58343229) assert approx_equal(flex.min(f_obs.data()), 0.00111552904935) assert approx_equal(flex.mean(f_obs.data()), 74.5148786464) assert f_obs.size() == f_calc.size() # define test set reflections flags = flex.bool(f_calc_partial.indices().size(), False) k = 0 for i in xrange(f_calc_partial.indices().size()): k = k + 1 if (k != 10): flags[i] = False else: k = 0 flags[i] = True assert flags.count(True) == 250 assert flags.count(False) == 2258 assert flags.size() == 2508 # *********************************************************TEST = 1 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=f_obs, f_calc=f_calc, free_reflections_per_bin=1000, flags=flags, interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.data().size() == beta.data().size() assert alpha.data().size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 0.914152454693) assert approx_equal(flex.max(alpha.data()), 0.914152454693) assert approx_equal(flex.min(beta.data()), 818.503411782) assert approx_equal(flex.max(beta.data()), 818.503411782) # *********************************************************TEST = 2 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=f_obs, f_calc=f_calc, free_reflections_per_bin=50, flags=flags, interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.data().size() == beta.data().size() assert alpha.data().size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 0.910350007113) assert approx_equal(flex.max(alpha.data()), 1.07104387776) assert approx_equal(flex.min(beta.data()), 21.7374310013) assert approx_equal(flex.max(beta.data()), 4222.81104745) # *********************************************************TEST = 3 alpha, beta = maxlik.alpha_beta_calc(f=f_obs, n_atoms_absent=100, n_atoms_included=900, bf_atoms_absent=25.0, final_error=0.0, absent_atom_type="C").alpha_beta() fom = max_lik.fom_and_phase_error( f_obs=f_obs.data(), f_model=flex.abs(f_calc.data()), alpha=alpha.data(), beta=beta.data(), epsilons=f_obs.epsilons().data().as_double(), centric_flags=f_obs.centric_flags().data()).fom() assert flex.max(fom) <= 1.0 assert flex.min(fom) >= 0.0 assert flex.min(alpha.data()) == flex.max(alpha.data()) == 1.0 assert approx_equal(flex.min(beta.data()), 7.964134920) assert approx_equal(flex.max(beta.data()), 13695.1589364) # *********************************************************TEST = 4 xs = crystal.symmetry((3, 4, 5), "P 2 2 2") mi = flex.miller_index(((1, -2, 3), (0, 0, -4))) ms = miller.set(xs, mi) fc = flex.double((1., 2.)) fo = flex.double((1., 2.)) mso = miller.set(xs, mi) mac = miller.array(ms, fc) mao = miller.array(ms, fo) alp = flex.double(2, 0.0) bet = flex.double(2, 1e+9) fom = max_lik.fom_and_phase_error( f_obs=mao.data(), f_model=mac.data(), alpha=alp, beta=bet, epsilons=mao.epsilons().data().as_double(), centric_flags=mao.centric_flags().data()).fom() assert approx_equal(fom, [0.0, 0.0]) alp = flex.double(2, 1.0) bet = flex.double(2, 1e+9) fom = max_lik.fom_and_phase_error( f_obs=mao.data(), f_model=mac.data(), alpha=alp, beta=bet, epsilons=mao.epsilons().data().as_double(), centric_flags=mao.centric_flags().data()).fom() assert approx_equal(fom, [0.0, 0.0]) alp = flex.double(2, 0.0) bet = flex.double(2, 1e-9) fom = max_lik.fom_and_phase_error( f_obs=mao.data(), f_model=mac.data(), alpha=alp, beta=bet, epsilons=mao.epsilons().data().as_double(), centric_flags=mao.centric_flags().data()).fom() assert approx_equal(fom, [0.0, 0.0]) alp = flex.double(2, 1.0) bet = flex.double(2, 1e-9) fom = max_lik.fom_and_phase_error( f_obs=mao.data(), f_model=mac.data(), alpha=alp, beta=bet, epsilons=mao.epsilons().data().as_double(), centric_flags=mao.centric_flags().data()).fom() assert approx_equal(fom, [1.0, 1.0])
def miller_array_export_as_shelx_hklf(miller_array, file_object=None, normalise_if_format_overflow=False, full_dynamic_range=False, scale_range=None): """\ For the full_dynamic_range option, normalise data outside the range that fits into 8.6g format, regardless of settings for normalise_if_format_overflow or scale_range Otherwise, if the maximum data value does not fit into the f8.2/f8.0 format: normalise_if_format_overflow=False: RuntimeError is thrown normalise_if_format_overflow=True: data is normalised to the largest number to fit f8.2/f8.0 format or within specified scale_range """ assert miller_array.is_real_array() if (file_object is None): file_object = sys.stdout def raise_f8_overflow(v): raise RuntimeError("SHELX HKL file F8.2/F8.0 format overflow: %.8g" % v) data = miller_array.data() sigmas = miller_array.sigmas() assert data is not None min_val = flex.min(data) max_val = flex.max(data) if (sigmas is not None): min_val = min(min_val, flex.min(sigmas)) max_val = max(max_val, flex.max(sigmas)) min_sc = 1 max_sc = 1 scale = 1 if full_dynamic_range: max_abs = 999999. max_val = max(abs(max_val), abs(min_val)) if (max_val > max_abs): scale = max_abs / max_val else: if scale_range is None: scale_range = (-999999., 9999999.) if (min_val < scale_range[0]): if not normalise_if_format_overflow: raise_f8_overflow(min_val) min_sc = scale_range[0] / min_val if (max_val > scale_range[1]): if (not normalise_if_format_overflow): raise_f8_overflow(max_val) max_sc = scale_range[1] / max_val scale = min(min_sc, max_sc) sigmas = miller_array.sigmas() s = 0.01 for i, h in enumerate(miller_array.indices()): if (sigmas is not None): s = sigmas[i] def fmt_3i4(h): result = "%4d%4d%4d" % h if (len(result) != 12): raise RuntimeError("SHELXL HKL file 3I4 format overflow: %s" % result) return result def fmt_f8(v): result = "%8.2f" % v if (len(result) != 8): result = "%8.1f" % v if (len(result) != 8): result = "%7d." % round(v) assert len(result) == 8 return result def fmt_fullrange_data(v): if (abs(v) >= 1.): result = "%8.6g" % v else: result = "%8.5f" % v return result def fmt_fullrange_sigma(v): if (abs(v) >= 1.): result = "%8.6g" % v elif (abs(v) < 0.00001): result = "%8.5f" % 0.00001 else: result = "%8.5f" % v return result if (full_dynamic_range): line = fmt_3i4(h) + fmt_fullrange_data( data[i] * scale) + fmt_fullrange_sigma(s * scale) else: line = fmt_3i4(h) + fmt_f8(data[i] * scale) + fmt_f8(s * scale) print(line, file=file_object) print(" 0 0 0 0.00 0.00", file=file_object)
def keywise_printout(data): for key in data: if key == 'ACTIVE_AREAS': print int(len(data[key]) / 4), "active areas, first one: ", list( data[key][0:4]) elif key == 'observations': print key, data[key], "Showing unit cell/spacegroup:" obs = data[key][0] uc = obs.unit_cell() uc.show_parameters() obs.space_group().info().show_summary() d = uc.d(obs.indices()) print "Number of observations:", len(obs.indices()) print "Max resolution: %f" % flex.min(d) print "Mean I/sigma:", flex.mean(obs.data()) / flex.mean( obs.sigmas()) print "I/sigma > 1 count:", (obs.data() / obs.sigmas() > 1).count(True) print "I <= 0:", len(obs.data().select(obs.data() <= 0)) from cctbx.crystal import symmetry sym = symmetry(unit_cell=uc, space_group=obs.space_group()) mset = sym.miller_set(indices=obs.indices(), anomalous_flag=False) binner = mset.setup_binner(n_bins=20) acceptable_resolution_bins = [] binned_avg_i_sigi = [] for i in binner.range_used(): d_max, d_min = binner.bin_d_range(i) sel = (d <= d_max) & (d > d_min) sel &= obs.data() > 0 intensities = obs.data().select(sel) sigmas = obs.sigmas().select(sel) n_refls = len(intensities) avg_i = flex.mean(intensities) if n_refls > 0 else 0 avg_i_sigi = flex.mean(intensities / sigmas) if n_refls > 0 else 0 acceptable_resolution_bins.append(avg_i_sigi >= 1.0) acceptable_resolution_bins = [ acceptable_resolution_bins[i] if False not in acceptable_resolution_bins[:i + 1] else False for i in range(len(acceptable_resolution_bins)) ] best_res = None for i, ok in zip(binner.range_used(), acceptable_resolution_bins): d_max, d_min = binner.bin_d_range(i) if ok: best_res = d_min else: break if best_res is None: print "Highest resolution with I/sigI >= 1.0: None" else: print "Highest resolution with I/sigI >= 1.0: %f" % d_min elif key == 'mapped_predictions': print key, data[key][0][0], "(only first shown of %d)" % len( data[key][0]) elif key == 'correction_vectors' and data[key] is not None and data[ key][0] is not None: if data[key][0] is None: print key, "None" else: print key, data[key][0][0], "(only first shown)" elif key == "DATA": print key, "len=%d max=%f min=%f dimensions=%s" % ( data[key].size(), flex.max(data[key]), flex.min( data[key]), str(data[key].focus())) elif key == "WAVELENGTH": print "WAVELENGTH", data[ key], ", converted to eV:", 12398.4187 / data[key] elif key == "fuller_kapton_absorption_correction": print key, data[key] if doplots: c = data[key][0] hist = flex.histogram(c, n_slots=30) from matplotlib import pyplot as plt plt.scatter(hist.slot_centers(), hist.slots()) plt.show() obs = data['observations'][0] preds = data['mapped_predictions'][0] p1 = preds.select(c == 1.0) p2 = preds.select((c != 1.0) & (c <= 1.5)) plt.scatter(preds.parts()[1], preds.parts()[0], c='g') plt.scatter(p1.parts()[1], p1.parts()[0], c='b') plt.scatter(p2.parts()[1], p2.parts()[0], c='r') plt.show() else: print key, data[key]
def get_uc_consensus(experiments_list, show_plot=False, return_only_first_indexed_model=False, finalize_method=None, clustering_params=None): ''' Uses the Rodriguez Laio 2014 method to do a clustering of the unit cells and then vote for the highest consensus unit cell. Input needs to be a list of experiments object. Clustering code taken from github.com/cctbx-xfel/cluster_regression Returns an experiment object with crystal unit cell from the cluster with the most points ''' if return_only_first_indexed_model: return [experiments_list[0].crystals()[0]], None cells = [] from xfel.clustering.singleframe import CellOnlyFrame save_plot = False # Flag for testing Lysozyme data from NKS.Make sure cluster_regression repository is present and configured # Program will exit after plots are displayed if this flag is true test_nks = False if test_nks: from cctbx import crystal import libtbx.load_env cluster_regression = libtbx.env.find_in_repositories( relative_path="cluster_regression", test=os.path.isdir) file_name = os.path.join(cluster_regression, 'examples', 'lysozyme1341.txt') for line in open(file_name, "r").xreadlines(): tokens = line.strip().split() unit_cell = tuple(float(x) for x in tokens[0:6]) space_group_symbol = tokens[6] crystal_symmetry = crystal.symmetry( unit_cell=unit_cell, space_group_symbol=space_group_symbol) cells.append(CellOnlyFrame(crystal_symmetry)) else: for experiment in experiments_list: if len(experiment.crystals()) > 1: print('IOTA:Should have only one crystal model') crystal_symmetry = experiment.crystals()[0].get_crystal_symmetry() cells.append(CellOnlyFrame(crystal_symmetry)) MM = [c.mm for c in cells] # metrical matrices MM_double = flex.double() for i in range(len(MM)): Tup = MM[i] for j in range(6): MM_double.append(Tup[j]) print('There are %d cells' % len(MM)) coord_x = flex.double([c.uc[0] for c in cells]) coord_y = flex.double([c.uc[1] for c in cells]) if show_plot or save_plot: import matplotlib if not show_plot: matplotlib.use('Agg') import matplotlib.pyplot as plt #from IPython import embed; embed(); exit() plt.plot([c.uc[0] for c in cells], [c.uc[1] for c in cells], "k.", markersize=3.) plt.axes().set_aspect("equal") if save_plot: plot_name = 'uc_cluster.png' plt.savefig(plot_name, size_inches=(10, 10), dpi=300, bbox_inches='tight') if show_plot: plt.show() print('Now constructing a Dij matrix: Starting Unit Cell clustering') NN = len(MM) from cctbx.uctbx.determine_unit_cell import NCDist_flatten Dij = NCDist_flatten(MM_double) d_c = flex.mean_and_variance( Dij.as_1d()).unweighted_sample_standard_deviation() #6.13 #FIXME should be a PHIL param if len(cells) < 5: return [experiments_list[0].crystals()[0]], None CM = clustering_manager(Dij=Dij, d_c=d_c, max_percentile_rho=0.95) n_cluster = 1 + flex.max(CM.cluster_id_final) print(len(cells), ' datapoints have been analyzed') print('%d CLUSTERS' % n_cluster) for i in range(n_cluster): item = flex.first_index(CM.cluster_id_maxima, i) print('Cluster %d central Unit cell = %d' % (i, item)) cells[item].crystal_symmetry.show_summary() # More plots for debugging appcolors = [ 'b', 'r', '#ff7f0e', '#2ca02c', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf' ] if show_plot: # Decision graph import matplotlib.pyplot as plt plt.plot(CM.rho, CM.delta, "r.", markersize=3.) for x in range(NN): if CM.cluster_id_maxima[x] >= 0: plt.plot([CM.rho[x]], [CM.delta[x]], "ro") plt.show() if show_plot: import matplotlib.pyplot as plt colors = [appcolors[i % 10] for i in CM.cluster_id_full] plt.scatter(coord_x, coord_y, marker='o', color=colors, linewidth=0.4, edgecolor='k') for i in range(n_cluster): item = flex.first_index(CM.cluster_id_maxima, i) plt.plot([cells[item].uc[0]], cells[item].uc[1], 'y.') plt.axes().set_aspect("equal") plt.show() if test_nks: exit() # Now look at each unit cell cluster for orientational clustering # idea is to cluster the orientational component in each of the unit cell clusters # do_orientational_clustering = not return_only_first_indexed_model # temporary. dxtbx_crystal_models = [] if do_orientational_clustering: print('IOTA: Starting orientational clustering') Dij_ori = {} # dictionary to store Dij for each cluster uc_experiments_list = { } # dictionary to store experiments_lists for each cluster from collections import Counter uc_cluster_count = Counter(list(CM.cluster_id_final)) # instantiate the Dij_ori flat 1-d array # Put all experiments list from same uc cluster together if True: from scitbx.matrix import sqr from cctbx_orientation_ext import crystal_orientation #crystal_orientation_list = [] #for i in range(len(experiments_list)): # crystal_orientation_list.append(crystal_orientation(experiments_list[i].crystals()[0].get_A(), True)) #from IPython import embed; embed(); exit() #A_direct = sqr(crystal_orientation_list[i].reciprocal_matrix()).transpose().inverse() #print ("Direct A matrix 1st element = %12.6f"%A_direct[0]) for i in range(len(experiments_list)): if CM.cluster_id_full[i] not in uc_experiments_list: uc_experiments_list[CM.cluster_id_full[i]] = [] uc_experiments_list[CM.cluster_id_full[i]].append( experiments_list[i]) for cluster in uc_cluster_count: # Make sure there are atleast a minimum number of samples in the cluster if uc_cluster_count[cluster] < 5: continue Dij_ori[cluster] = flex.double( [[0.0] * uc_cluster_count[cluster]] * uc_cluster_count[cluster]) # Now populate the Dij_ori array N_samples_in_cluster = len(uc_experiments_list[cluster]) for i in range(N_samples_in_cluster - 1): for j in range(i + 1, N_samples_in_cluster): dij_ori = get_dij_ori( uc_experiments_list[cluster][i].crystals()[0], uc_experiments_list[cluster][j].crystals()[0]) Dij_ori[cluster][N_samples_in_cluster * i + j] = dij_ori Dij_ori[cluster][N_samples_in_cluster * j + i] = dij_ori # Now do the orientational cluster analysis #from IPython import embed; embed(); exit() d_c_ori = 0.13 from exafel_project.ADSE13_25.clustering.plot_with_dimensional_embedding import plot_with_dimensional_embedding #plot_with_dimensional_embedding(1-Dij_ori[1]/flex.max(Dij_ori[1]), show_plot=True) for cluster in Dij_ori: d_c_ori = flex.mean_and_variance(Dij_ori[cluster].as_1d( )).unweighted_sample_standard_deviation() CM_ori = clustering_manager(Dij=Dij_ori[cluster], d_c=d_c_ori, max_percentile_rho=0.85) n_cluster_ori = 1 + flex.max(CM_ori.cluster_id_final) #from IPython import embed; embed() #FIXME should be a PHIL param for i in range(n_cluster_ori): if len([zz for zz in CM_ori.cluster_id_final if zz == i]) < 5: continue item = flex.first_index(CM_ori.cluster_id_maxima, i) dxtbx_crystal_model = uc_experiments_list[cluster][ item].crystals()[0] dxtbx_crystal_models.append(dxtbx_crystal_model) from scitbx.matrix import sqr from cctbx_orientation_ext import crystal_orientation crystal_orientation = crystal_orientation( dxtbx_crystal_model.get_A(), True) A_direct = sqr(crystal_orientation.reciprocal_matrix() ).transpose().inverse() print( "IOTA: Direct A matrix 1st element of orientational cluster %d = %12.6f" % (i, A_direct[0])) if show_plot: # Decision graph stretch_plot_factor = 1.05 # (1+fraction of limits by which xlim,ylim should be set) import matplotlib.pyplot as plt plt.plot(CM_ori.rho, CM_ori.delta, "r.", markersize=3.) for x in range(len(list(CM_ori.cluster_id_final))): if CM_ori.cluster_id_maxima[x] >= 0: plt.plot([CM_ori.rho[x]], [CM_ori.delta[x]], "ro") #from IPython import embed; embed(); exit() plt.xlim([-10, stretch_plot_factor * flex.max(CM_ori.rho)]) plt.ylim([-10, stretch_plot_factor * flex.max(CM_ori.delta)]) plt.show() # Make sure the crystal models are not too close to each other # FIXME should be a PHIL min_angle = 5.0 # taken from indexer.py close_models_list = [] if len(dxtbx_crystal_models) > 1: from dials.algorithms.indexing.compare_orientation_matrices import difference_rotation_matrix_axis_angle for i_a in range(0, len(dxtbx_crystal_models) - 1): for i_b in range(i_a, len(dxtbx_crystal_models)): cryst_a = dxtbx_crystal_models[i_a] cryst_b = dxtbx_crystal_models[i_b] R_ab, axis, angle, cb_op_ab = difference_rotation_matrix_axis_angle( cryst_a, cryst_b) # FIXME if abs(angle) < min_angle: # degrees close_models_list.append((i_a, i_b)) # Now prune the dxtbx_crystal_models list for close_models in close_models_list: i_a, i_b = close_models if dxtbx_crystal_models[i_a] is not None and dxtbx_crystal_models[ i_b] is not None: dxtbx_crystal_models[i_a] = None dxtbx_crystal_models = [x for x in dxtbx_crystal_models if x is not None] if len(dxtbx_crystal_models) > 0: return dxtbx_crystal_models, None else: # If nothing works, atleast return the 1st crystal model that was found return [experiments_list[0].crystals()[0]], None
def exercise(space_group_info, anomalous_flag, use_u_aniso, n_elements=3, d_min=3., verbose=0): structure_z = random_structure.xray_structure( space_group_info, elements=("Se", ) * n_elements, volume_per_atom=200, random_f_prime_d_min=1.0, random_f_double_prime=anomalous_flag, random_u_iso=True, use_u_aniso=use_u_aniso, random_occupancy=True) check_weight_without_occupancy(structure_z) f_z = structure_z.structure_factors(anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct").f_calc() f_abs_z = abs(f_z) f_rad_z = f_z.phases() f_deg_z = f_z.phases(deg=True) hl_z = generate_random_hl(miller_set=f_z) hl_z_rad = hl_z.phase_integrals() if (0 or verbose): structure_z.show_summary().show_scatterers() print "n_special_positions:", \ structure_z.special_position_indices().size() z2p_op = structure_z.space_group().z2p_op() z2p_op = sgtbx.change_of_basis_op(z2p_op.c() + sgtbx.tr_vec( (2, -1, 3), 12).new_denominator(z2p_op.c().t().den())) for change_hand in [False, True]: if (change_hand): z2p_op = z2p_op * sgtbx.change_of_basis_op("-x,-y,-z") structure_p = structure_z.change_basis(z2p_op) check_weight_without_occupancy(structure_p) check_site_symmetry_table(structure_z, z2p_op, structure_p) if (0 or verbose): structure_p.show_summary().show_scatterers() print "n_special_positions:", \ structure_p.special_position_indices().size() assert tuple(structure_p.special_position_indices()) \ == tuple(structure_z.special_position_indices()) structure_pz = structure_p.change_basis(z2p_op.inverse()) check_weight_without_occupancy(structure_pz) check_site_symmetry_table(structure_p, z2p_op.inverse(), structure_pz) assert structure_pz.unit_cell().is_similar_to(structure_z.unit_cell()) assert structure_pz.space_group() == structure_z.space_group() f_pz = f_z.structure_factors_from_scatterers( xray_structure=structure_pz, algorithm="direct").f_calc() f_abs_pz = abs(f_pz) f_rad_pz = f_pz.phases() f_deg_pz = f_pz.phases(deg=True) c = flex.linear_correlation(f_abs_z.data(), f_abs_pz.data()) assert c.is_well_defined() if (0 or verbose): print "correlation:", c.coefficient() assert c.coefficient() > 0.999 f_p_cb = f_z.change_basis(z2p_op) f_abs_p_cb = f_abs_z.change_basis(z2p_op) f_rad_p_cb = f_rad_z.change_basis(z2p_op, deg=False) f_deg_p_cb = f_deg_z.change_basis(z2p_op, deg=True) hl_p_cb = hl_z.change_basis(z2p_op) hl_p_cb_rad = hl_p_cb.phase_integrals() assert approx_equal( hl_z_rad.change_basis(z2p_op).data(), hl_p_cb_rad.data()) assert f_abs_p_cb.indices().all_eq(f_p_cb.indices()) if (not change_hand): o = flex.order(f_abs_p_cb.indices(), f_abs_z.indices()) else: o = flex.order(-f_abs_p_cb.indices(), f_abs_z.indices()) if (f_abs_z.space_group().n_ltr() == 1): assert o == 0 else: assert o != 0 f_pz = f_p_cb.change_basis(z2p_op.inverse()) f_abs_pz = f_abs_p_cb.change_basis(z2p_op.inverse()) f_rad_pz = f_rad_p_cb.change_basis(z2p_op.inverse(), deg=False) f_deg_pz = f_deg_p_cb.change_basis(z2p_op.inverse(), deg=True) hl_pz = hl_p_cb.change_basis(z2p_op.inverse()) hl_pz_rad = hl_pz.phase_integrals() assert approx_equal(hl_z_rad.data(), hl_pz_rad.data()) for i, o in zip(hl_z.data(), hl_pz.data()): assert approx_equal(i, o) assert approx_equal(flex.max(flex.abs(f_pz.data() - f_z.data())), 0) assert flex.order(f_abs_pz.indices(), f_abs_z.indices()) == 0 assert f_abs_pz.indices().all_eq(f_pz.indices()) assert approx_equal( flex.max( scitbx.math.phase_error(phi1=f_rad_pz.data(), phi2=f_rad_z.data())), 0) assert approx_equal( flex.max( scitbx.math.phase_error(phi1=f_deg_pz.data(), phi2=f_deg_z.data(), deg=True)), 0) assert approx_equal(f_deg_pz.data(), f_rad_pz.data() * (180 / math.pi)) f_p_sf = f_p_cb.structure_factors_from_scatterers( xray_structure=structure_p, algorithm="direct").f_calc() det = abs(z2p_op.c().r().determinant()) assert approx_equal( flex.max(flex.abs(f_p_sf.data() * complex(det) - f_p_cb.data())), 0) f_abs_p_sf = abs(f_p_sf) f_rad_p_sf = f_p_sf.phases() f_deg_p_sf = f_p_sf.phases(deg=True) assert approx_equal( flex.max( scitbx.math.phase_error(phi1=f_rad_p_sf.data(), phi2=f_rad_p_cb.data())), 0) assert approx_equal( flex.max( scitbx.math.phase_error(phi1=f_deg_p_sf.data(), phi2=f_deg_p_cb.data(), deg=True)), 0) c = flex.linear_correlation(f_abs_p_sf.data(), f_abs_p_cb.data()) assert c.is_well_defined() if (0 or verbose): print "correlation:", c.coefficient() assert c.coefficient() > 0.999