def __init__(self, sweep, params): self.params = params flex.set_random_seed(params.random_seed) unit_cell = params.unit_cell assert unit_cell is not None sgi = params.space_group if sgi is None: sgi = sgtbx.space_group_info(symbol="P 1") B = scitbx.matrix.sqr(unit_cell.fractionalization_matrix()).transpose() U = scitbx.matrix.sqr(flex.random_double_r3_rotation_matrix()) direct_matrix = (U * B).inverse() crystal_model = Crystal(direct_matrix[0:3], direct_matrix[3:6], direct_matrix[6:9], space_group=sgi.group()) scan = sweep.get_scan() angle = self.params.rotation_angle scan.set_image_range((1, iceil(angle/scan.get_oscillation()[1]))) predicted = predict_reflections(sweep, crystal_model) beam_vectors = predicted.beam_vector() S = beam_vectors - sweep.get_beam().get_s0() centroids = S.rotate_around_origin(sweep.get_goniometer().get_rotation_axis(), -predicted.rotation_angle()) self.d_min = self.params.reciprocal_space_grid.d_min self.gridding = tuple([self.params.reciprocal_space_grid.n_points]*3) centroids = centroids.select((1/centroids.norms())>=self.d_min) assert len(centroids) > 0 self.map_to_grid(sweep, centroids) self.fft() debug_write_reciprocal_lattice_points_as_pdb(centroids) self.debug_write_ccp4_map(self.grid_real, "fft.map")
def exercise_f_model_no_scales(symbol = "C 2"): random.seed(0) flex.set_random_seed(0) x = random_structure.xray_structure( space_group_info = sgtbx.space_group_info(symbol=symbol), elements =(("O","N","C")*5+("H",)*10), volume_per_atom = 200, min_distance = 1.5, general_positions_only = True, random_u_iso = True, random_occupancy = False) f_obs = abs(x.structure_factors(d_min = 1.5, algorithm="fft").f_calc()) x.shake_sites_in_place(mean_distance=1) k_iso = flex.double(f_obs.data().size(), 2) k_aniso = flex.double(f_obs.data().size(), 3) fmodel = mmtbx.f_model.manager( xray_structure = x, k_isotropic = k_iso, k_anisotropic = k_aniso, f_obs = f_obs) fc = abs(fmodel.f_calc()).data() fm = abs(fmodel.f_model()).data() fmns = abs(fmodel.f_model_no_scales()).data() assert approx_equal(flex.mean(fm/fc), 6) assert approx_equal(flex.mean(fmns/fc), 1)
def __init__(self, sweep, params): self.params = params flex.set_random_seed(params.random_seed) unit_cell = params.unit_cell assert unit_cell is not None sgi = params.space_group if sgi is None: sgi = sgtbx.space_group_info(symbol="P 1") B = scitbx.matrix.sqr(unit_cell.fractionalization_matrix()).transpose() U = scitbx.matrix.sqr(flex.random_double_r3_rotation_matrix()) direct_matrix = (U * B).inverse() crystal_model = Crystal( direct_matrix[0:3], direct_matrix[3:6], direct_matrix[6:9], space_group=sgi.group(), ) scan = sweep.get_scan() angle = self.params.rotation_angle scan.set_image_range((1, iceil(angle / scan.get_oscillation()[1]))) predicted = predict_reflections(sweep, crystal_model) beam_vectors = predicted.beam_vector() S = beam_vectors - sweep.get_beam().get_s0() centroids = S.rotate_around_origin( sweep.get_goniometer().get_rotation_axis(), -predicted.rotation_angle() ) self.d_min = self.params.reciprocal_space_grid.d_min self.gridding = tuple([self.params.reciprocal_space_grid.n_points] * 3) centroids = centroids.select((1 / centroids.norms()) >= self.d_min) assert len(centroids) > 0 self.map_to_grid(sweep, centroids) self.fft() debug_write_reciprocal_lattice_points_as_pdb(centroids) self.debug_write_ccp4_map(self.grid_real, "fft.map")
def run(): libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser() .option(None, "--fix_random_seeds", action="store_true", default=False) .option(None, "--runs", type='int', default=1) .option(None, "--verbose", action="store_true", default=False) .option(None, "--skip-twin-test", dest='skip_twin_test', action="store_true", default=False) ).process(args=sys.argv[1:]) if command_line.options.fix_random_seeds: flex.set_random_seed(1) random.seed(2) n_runs = command_line.options.runs if n_runs > 1: refinement_test.ls_cycle_repeats = n_runs exercise_normal_equations() exercise_floating_origin_dynamic_weighting(command_line.options.verbose) special_positions_test(n_runs).run() if not command_line.options.skip_twin_test: twin_test().run()
def run(args): random_seed = None forever = False for arg in args: if (arg.startswith("--random_seed=")): random_seed = int(arg.split("=", 1)[1]) if (arg.startswith("--forever")): forever = True if forever: assert random_seed is None if random_seed is not None: assert forever == False while 1: if (random_seed is None): random_seed = flex.get_random_seed() print "random_seed:", random_seed sys.stdout.flush() random.seed(random_seed) flex.set_random_seed(value=random_seed) calculate_fobs() exercise_1() exercise_2() check_result() sys.stdout.flush() if(not forever): print "random_seed last used:", random_seed break
def run(): libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser() .option(None, "--fix_random_seeds", action="store_true", default=False) .option(None, "--runs", type='int', default=1) .option(None, "--verbose", action="store_true", default=False) ).process(args=sys.argv[1:]) if command_line.options.fix_random_seeds: flex.set_random_seed(1) random.seed(2) n_runs = command_line.options.runs if n_runs > 1: refinement_test.ls_cycle_repeats = n_runs exercise_normal_equations() exercise_floating_origin_dynamic_weighting(command_line.options.verbose) special_positions_test(n_runs).run() twin_test().run()
def get_sf(k_sol, b_sol, b_cart, xrs, miller_set=None, d_min=None, twin_law=None, sfg_params=None): random.seed(0) flex.set_random_seed(0) if (miller_set is None): assert d_min is not None f_dummy = abs( xrs.structure_factors(d_min=d_min, anomalous_flag=False).f_calc()) else: f_dummy = miller_set assert d_min is None r_free_flags = f_dummy.generate_r_free_flags(fraction=0.1) fmodel = mmtbx.f_model.manager(r_free_flags=r_free_flags, f_obs=f_dummy, sf_and_grads_accuracy_params=sfg_params, xray_structure=xrs, twin_law=twin_law) ss = 1. / flex.pow2(r_free_flags.d_spacings().data()) / 4. k_mask = mmtbx.f_model.ext.k_mask(ss, k_sol, b_sol) u_star = adptbx.u_cart_as_u_star(xrs.unit_cell(), adptbx.b_as_u(b_cart)) k_anisotropic = mmtbx.f_model.ext.k_anisotropic(r_free_flags.indices(), u_star) fmodel.update_xray_structure(xray_structure=xrs, update_f_calc=True, update_f_mask=True) fmodel.update_core(k_mask=k_mask, k_anisotropic=k_anisotropic) f_obs = abs(fmodel.f_model()) return f_obs, r_free_flags
def run(args): random_seed = None forever = False for arg in args: if (arg.startswith("--random_seed=")): random_seed = int(arg.split("=", 1)[1]) if (arg.startswith("--forever")): forever = True if forever: assert random_seed is None if random_seed is not None: assert forever == False while 1: if (random_seed is None): random_seed = flex.get_random_seed() print "random_seed:", random_seed sys.stdout.flush() random.seed(random_seed) flex.set_random_seed(value=random_seed) calculate_fobs() exercise_1() exercise_2() check_result() sys.stdout.flush() if (not forever): print "random_seed last used:", random_seed break
def exercise(flags, space_group_info): if not flags.repeats: flags.repeats = 1 if not flags.algo: flags.algo = "weak_reflection_improved" if not flags.on: flags.on = "E" if flags.fix_seed: random.seed(1) flex.set_random_seed(1) n = len(space_group_info.group()) print space_group_info.type().hall_symbol(), if not flags.high_symmetry and n > 24: print ' [ skipped ]' if flags.Verbose: print return else: print n_C = 12 // n or 1 n_O = 6 // n n_N = 3 // n if flags.Verbose: print "unit cell content: C%i O%i N%i" % (n_C * n, n_O * n, n_N * n) print "asu content: C%i O%i N%i" % (n_C, n_O, n_N) print "on %s's with %s" % (flags.on, flags.algo) flipping_type = eval("charge_flipping.%s_iterator" % flags.algo) for i in xrange(int(flags.repeats)): randomly_exercise(flipping_type=flipping_type, space_group_info=space_group_info, elements=["C"] * n_C + ["O"] * n_O + ["N"] * n_N, anomalous_flag=False, d_min=0.8, verbose=flags.Verbose, amplitude_type=flags.on) if flags.Verbose: print
def exercise(): import random if (1): # fixed random seed to avoid rare failures random.seed(0) flex.set_random_seed(0) verbose = "--verbose" in sys.argv[1:] for n in [4, 10, 20, 100, 200]: exercise_core(n=n, verbose=verbose) print "OK"
def run(args): if "--fix_random_seeds" in args: random.seed(1) flex.set_random_seed(1) scitbx.random.set_random_seed(1) libtbx.utils.show_times_at_exit() for i in range(10): exercise_optimise_shelxl_weights() exercise_weighting_schemes()
def exercise(): import random if 1: # fixed random seed to avoid rare failures random.seed(0) flex.set_random_seed(0) verbose = "--verbose" in sys.argv[1:] for n in [4, 10, 20, 100, 200]: exercise_core(n=n, verbose=verbose) print "OK"
def exercise_unmerged () : quartz_structure = xray.structure( special_position_settings=crystal.special_position_settings( crystal_symmetry=crystal.symmetry( unit_cell=(5.01,5.01,5.47,90,90,120), space_group_symbol="P6222")), scatterers=flex.xray_scatterer([ xray.scatterer( label="Si", site=(1/2.,1/2.,1/3.), u=0.2), xray.scatterer( label="O", site=(0.197,-0.197,0.83333), u=0.1)])) quartz_structure.set_inelastic_form_factors( photon=1.54, table="sasaki") fc = abs(quartz_structure.structure_factors(d_min=1.0).f_calc()) symm = fc.crystal_symmetry() icalc = fc.expand_to_p1().f_as_f_sq().set_observation_type_xray_intensity() # generate 'unmerged' data i_obs = icalc.customized_copy(crystal_symmetry=symm) # now make up sigmas and some (hopefully realistic) error flex.set_random_seed(12345) n_refl = i_obs.size() sigmas = flex.random_double(n_refl) * flex.mean(fc.data()) sigmas = icalc.customized_copy(data=sigmas).apply_debye_waller_factors( u_iso=0.15) err = (flex.double(n_refl, 0.5) - flex.random_double(n_refl)) * 2 i_obs = i_obs.customized_copy( sigmas=sigmas.data(), data=i_obs.data() + err) # check for unmerged acentrics assert i_obs.is_unmerged_intensity_array() i_obs_centric = i_obs.select(i_obs.centric_flags().data()) i_obs_acentric = i_obs.select(~(i_obs.centric_flags().data())) i_mrg_acentric = i_obs_acentric.merge_equivalents().array() i_mixed = i_mrg_acentric.concatenate(i_obs_centric) assert not i_mixed.is_unmerged_intensity_array() # XXX These results of these functions are heavily dependent on the # behavior of the random number generator, which is not consistent across # platforms - therefore we can only check for very approximate values. # Exact numerical results are tested with real data (stored elsewhere). # CC1/2, etc. assert approx_equal(i_obs.cc_one_half(), 0.9998, eps=0.001) assert i_obs.resolution_filter(d_max=1.2).cc_one_half() > 0 assert i_obs.cc_anom() > 0.1 r_ano = i_obs.r_anom() assert approx_equal(r_ano, 0.080756, eps=0.0001) # merging stats i_mrg = i_obs.merge_equivalents() assert i_mrg.r_merge() < 0.1 assert i_mrg.r_meas() < 0.1 assert i_mrg.r_pim() < 0.05
def run_call_back(flags, space_group_info): if not space_group_info.group().is_centric(): if flags.fix_random_seeds: random.seed(1) flex.set_random_seed(1) scitbx.random.set_random_seed(1) hooft_analysis_test_case( space_group_info,use_students_t_errors=False).exercise(debug=flags.Debug) if students_t_available: hooft_analysis_test_case( space_group_info,use_students_t_errors=True).exercise(debug=flags.Debug)
def exercise(self, xray_structure=None, space_group_info=None, verbose=False, fixed_random_seed=True, **kwds): assert [xray_structure, space_group_info].count(None) == 1 if xray_structure is None: self.xs = self.random_structure(space_group_info, set_grads=True) else: self.xs = xray_structure if fixed_random_seed: random.seed(1) flex.set_random_seed(1) self.do_exercise(verbose)
def __init__(self, xray_structure, params, pdb_hierarchy, log=None): self.log = log self.pdb_hierarchy = pdb_hierarchy self.crystal_symmetry = xray_structure.crystal_symmetry() if (self.log is None): self.log = sys.stdout self.xray_structure = xray_structure self.params = params asc = self.pdb_hierarchy.atom_selection_cache( special_position_settings=crystal.special_position_settings( crystal_symmetry=self.crystal_symmetry)) if (self.params.random_seed is not None): random.seed(self.params.random_seed) flex.set_random_seed(self.params.random_seed) self.top_selection = flex.smart_selection( flags=flex.bool(xray_structure.scatterers().size(), True)) if (self.params.selection is not None): self.top_selection = flex.smart_selection( flags=asc.selection(self.params.selection)) self._rotate_about_axis() self._process_adp() self._process_sites() self._process_occupancies() self._put_in_box() self._change_of_basis() # Up to this point we are done with self.xray_structure self.pdb_hierarchy.adopt_xray_structure(self.xray_structure) # Now only manipulations that use self.pdb_hierarchy are done if (self.params.set_chemical_element_simple_if_necessary or self.params.rename_chain_id.old_id or self.params.renumber_residues or self.params.increment_resseq or self.params.convert_semet_to_met or self.params.convert_met_to_semet or self.params.set_charge.charge or self.params.truncate_to_polyala or self.params.truncate_to_polygly or self.params.remove_alt_confs or self.params.move_waters_last or self.params.remove_fraction or self.params.keep or self.params.remove): del self.xray_structure # it is invalide below this point self._set_chemical_element_simple_if_necessary() self._rename_chain_id() self._renumber_residues() self._convert_semet_to_met() self._convert_met_to_semet() self._set_atomic_charge() self._truncate_to_poly_ala() self._truncate_to_poly_gly() self._remove_alt_confs() self._move_waters() self._remove_atoms() self._apply_keep_remove()
def exercise_unmerged(): quartz_structure = xray.structure( special_position_settings=crystal.special_position_settings( crystal_symmetry=crystal.symmetry(unit_cell=(5.01, 5.01, 5.47, 90, 90, 120), space_group_symbol="P6222")), scatterers=flex.xray_scatterer([ xray.scatterer(label="Si", site=(1 / 2., 1 / 2., 1 / 3.), u=0.2), xray.scatterer(label="O", site=(0.197, -0.197, 0.83333), u=0.1) ])) quartz_structure.set_inelastic_form_factors(photon=1.54, table="sasaki") fc = abs(quartz_structure.structure_factors(d_min=1.0).f_calc()) symm = fc.crystal_symmetry() icalc = fc.expand_to_p1().f_as_f_sq().set_observation_type_xray_intensity() # generate 'unmerged' data i_obs = icalc.customized_copy(crystal_symmetry=symm) # now make up sigmas and some (hopefully realistic) error flex.set_random_seed(12345) n_refl = i_obs.size() sigmas = flex.random_double(n_refl) * flex.mean(fc.data()) sigmas = icalc.customized_copy(data=sigmas).apply_debye_waller_factors( u_iso=0.15) err = (flex.double(n_refl, 0.5) - flex.random_double(n_refl)) * 2 i_obs = i_obs.customized_copy(sigmas=sigmas.data(), data=i_obs.data() + err) # check for unmerged acentrics assert i_obs.is_unmerged_intensity_array() i_obs_centric = i_obs.select(i_obs.centric_flags().data()) i_obs_acentric = i_obs.select(~(i_obs.centric_flags().data())) i_mrg_acentric = i_obs_acentric.merge_equivalents().array() i_mixed = i_mrg_acentric.concatenate(i_obs_centric) assert not i_mixed.is_unmerged_intensity_array() # XXX These results of these functions are heavily dependent on the # behavior of the random number generator, which is not consistent across # platforms - therefore we can only check for very approximate values. # Exact numerical results are tested with real data (stored elsewhere). # CC1/2, etc. assert approx_equal(i_obs.cc_one_half(), 0.9999, eps=0.001) assert approx_equal(i_obs.cc_one_half_sigma_tau(), 0.9999, eps=0.001) assert i_obs.resolution_filter(d_max=1.2).cc_one_half() > 0 assert i_obs.cc_anom() > 0.1 r_ano = i_obs.r_anom() assert approx_equal(r_ano, 0.080756, eps=0.0001) # merging stats i_mrg = i_obs.merge_equivalents() assert i_mrg.r_merge() < 0.1 assert i_mrg.r_meas() < 0.1 assert i_mrg.r_pim() < 0.05
def exercise_split_unmerged(): import random random.seed(42) flex.set_random_seed(42) from cctbx import crystal base_set = miller.build_set( crystal_symmetry=crystal.symmetry( unit_cell=(10,10,10,90,90,90), space_group_symbol="P1"), d_min=1.6, anomalous_flag=False) indices = base_set.indices() assert (len(indices) == 510) unmerged_hkl = flex.miller_index() unmerged_data = flex.double() unmerged_sigmas = flex.double() redundancies = flex.size_t() # XXX grossly overengineered, but I wanted to get a realistic CC to make sure # the reflections are being split properly for i, hkl in enumerate(indices): n_obs = min(8, 1 + i % 12) redundancies.append(n_obs) intensity_merged = (510 - i) + (510 % 27) for j in range(n_obs): unmerged_hkl.append(hkl) intensity = intensity_merged + 20 * (510 % (7 * (j+1))) sigma = max(0.5, i % 10) unmerged_data.append(intensity) unmerged_sigmas.append(sigma) assert (unmerged_hkl.size() == 2877) unmerged_array = miller.set( crystal_symmetry=base_set, indices=unmerged_hkl, anomalous_flag=False).array(data=unmerged_data, sigmas=unmerged_sigmas) split = miller.split_unmerged( unmerged_indices=unmerged_hkl, unmerged_data=unmerged_data, unmerged_sigmas=unmerged_sigmas) assert (split.data_1.size() == split.data_2.size() == 467) cc = miller.compute_cc_one_half(unmerged_array) assert approx_equal(cc, 0.861, eps=0.001) unmerged_array.setup_binner(n_bins=10) unmerged_array.set_observation_type_xray_intensity() result = unmerged_array.cc_one_half(use_binning=True) assert approx_equal( result.data[1:-1], [0.549, 0.789, 0.843, 0.835, 0.863, 0.860, 0.893, 0.847, 0.875, 0.859], eps=0.05)
def exercise_split_unmerged () : import random random.seed(42) flex.set_random_seed(42) from cctbx import crystal base_set = miller.build_set( crystal_symmetry=crystal.symmetry( unit_cell=(10,10,10,90,90,90), space_group_symbol="P1"), d_min=1.6, anomalous_flag=False) indices = base_set.indices() assert (len(indices) == 510) unmerged_hkl = flex.miller_index() unmerged_data = flex.double() unmerged_sigmas = flex.double() redundancies = flex.size_t() # XXX grossly overengineered, but I wanted to get a realistic CC to make sure # the reflections are being split properly for i, hkl in enumerate(indices) : n_obs = min(8, 1 + i % 12) redundancies.append(n_obs) intensity_merged = (510 - i) + (510 % 27) for j in range(n_obs) : unmerged_hkl.append(hkl) intensity = intensity_merged + 20 * (510 % (7 * (j+1))) sigma = max(0.5, i % 10) unmerged_data.append(intensity) unmerged_sigmas.append(sigma) assert (unmerged_hkl.size() == 2877) unmerged_array = miller.set( crystal_symmetry=base_set, indices=unmerged_hkl, anomalous_flag=False).array(data=unmerged_data, sigmas=unmerged_sigmas) split = miller.split_unmerged( unmerged_indices=unmerged_hkl, unmerged_data=unmerged_data, unmerged_sigmas=unmerged_sigmas) assert (split.data_1.size() == split.data_2.size() == 467) cc = miller.compute_cc_one_half(unmerged_array) assert approx_equal(cc, 0.861, eps=0.001) unmerged_array.setup_binner(n_bins=10) unmerged_array.set_observation_type_xray_intensity() result = unmerged_array.cc_one_half(use_binning=True) assert approx_equal( result.data[1:-1], [0.549, 0.789, 0.843, 0.835, 0.863, 0.860, 0.893, 0.847, 0.875, 0.859], eps=0.05)
def exercise_4_f_hydrogens(): for d_min in [1, 2, 3]: for q in [0, 0.9, 1]: random.seed(0) flex.set_random_seed(0) x = random_structure.xray_structure( space_group_info=sgtbx.space_group_info("P 4"), elements=(("O", "N", "C") * 5 + ("H", ) * 95), volume_per_atom=200, min_distance=1.5, general_positions_only=True, random_u_iso=True, random_occupancy=False) hd_sel = x.hd_selection() b_isos = x.select( ~hd_sel).extract_u_iso_or_u_equiv() * adptbx.u_as_b(1.) mmm = b_isos.min_max_mean().as_tuple() b_mean = int(mmm[2]) x = x.set_b_iso(value=b_mean, selection=hd_sel) x.scattering_type_registry(table="wk1995") x.set_occupancies(value=q, selection=hd_sel) fc = x.structure_factors(d_min=d_min, algorithm="direct").f_calc() f_obs = abs(fc) x = x.deep_copy_scatterers() x.set_occupancies(value=0.0, selection=x.hd_selection()) sfg_params = mmtbx.f_model.sf_and_grads_accuracy_master_params.extract( ) sfg_params.algorithm = "direct" r_free_flags = f_obs.generate_r_free_flags(fraction=0.1) fmodel = mmtbx.f_model.manager( xray_structure=x, f_obs=f_obs, r_free_flags=r_free_flags, sf_and_grads_accuracy_params=sfg_params) if (q == 0): assert approx_equal(fmodel.r_work(), 0) else: assert fmodel.r_work() > 0.05, fmodel.r_work() params = bss.master_params.extract() params.bulk_solvent = False params.anisotropic_scaling = False o = fmodel.update_all_scales(fast=False, params=params) assert approx_equal(o.k_sol[0], 0) assert approx_equal(o.b_sol[0], 0) assert approx_equal(o.b_cart, [0, 0, 0, 0, 0, 0]) assert approx_equal(o.k_h, q) assert approx_equal(fmodel.r_work(), 0)
def exercise_4_f_hydrogens(): for d_min in [1,2,3]: for q in [0, 0.9, 1]: random.seed(0) flex.set_random_seed(0) x = random_structure.xray_structure( space_group_info = sgtbx.space_group_info("P 4"), elements =(("O","N","C")*5 + ("H",)*95), volume_per_atom = 200, min_distance = 1.5, general_positions_only = True, random_u_iso = True, random_occupancy = False) hd_sel = x.hd_selection() b_isos = x.select(~hd_sel).extract_u_iso_or_u_equiv()*adptbx.u_as_b(1.) mmm = b_isos.min_max_mean().as_tuple() b_mean = int(mmm[2]) x = x.set_b_iso(value=b_mean, selection = hd_sel) x.scattering_type_registry(table="wk1995") x.set_occupancies(value=q, selection = hd_sel) fc = x.structure_factors(d_min = d_min, algorithm="direct").f_calc() f_obs = abs(fc) x = x.deep_copy_scatterers() x.set_occupancies(value=0.0, selection = x.hd_selection()) sfg_params = mmtbx.f_model.sf_and_grads_accuracy_master_params.extract() sfg_params.algorithm = "direct" r_free_flags = f_obs.generate_r_free_flags(fraction = 0.1) fmodel = mmtbx.f_model.manager( xray_structure = x, f_obs = f_obs, r_free_flags = r_free_flags, sf_and_grads_accuracy_params = sfg_params) if(q==0): assert approx_equal(fmodel.r_work(), 0) else: assert fmodel.r_work() > 0.05, fmodel.r_work() params = bss.master_params.extract() params.bulk_solvent=False params.anisotropic_scaling=False o = fmodel.update_all_scales(fast=False, params=params) assert approx_equal(o.k_sol[0],0) assert approx_equal(o.b_sol[0],0) assert approx_equal(o.b_cart,[0,0,0,0,0,0]) assert approx_equal(o.k_h, q) assert approx_equal(fmodel.r_work(), 0)
def exercise(args): forever = False random_seed = None for arg in args: if (arg == "--forever"): forever = True elif (arg.startswith("--random_seed=")): random_seed = int(arg.split("=", 1)[1]) if (random_seed is None): random_seed = flex.get_random_seed() while True: print "random_seed:", random_seed random.seed(random_seed) flex.set_random_seed(value=random_seed) exercise_2() if (not forever): break random_seed += 1 print format_cpu_times()
def exercise(args): forever = False random_seed = None for arg in args: if (arg == "--forever"): forever = True elif (arg.startswith("--random_seed=")): random_seed = int(arg.split("=", 1)[1]) if (random_seed is None): random_seed = flex.get_random_seed() while True: print("random_seed:", random_seed) random.seed(random_seed) flex.set_random_seed(value=random_seed) exercise_2() if (not forever): break random_seed += 1 print(format_cpu_times())
def exercise_top_largest_f_obs_f_model_differences(threshold_percent=10, symbol="C 2"): random.seed(0) flex.set_random_seed(0) x = random_structure.xray_structure( space_group_info=sgtbx.space_group_info(symbol=symbol), elements=(("O", "N", "C") * 5 + ("H", ) * 10), volume_per_atom=200, min_distance=1.5, general_positions_only=True, random_u_iso=True, random_occupancy=False) f_obs = abs(x.structure_factors(d_min=1.5, algorithm="fft").f_calc()) x.shake_sites_in_place(mean_distance=1) fmodel = mmtbx.f_model.manager(xray_structure=x, f_obs=f_obs) fmodel.update_all_scales(update_f_part1=False) v, d = fmodel.top_largest_f_obs_f_model_differences( threshold_percent=threshold_percent) n = (d > v).count(True) * 100. / d.size() assert approx_equal(threshold_percent, n, 0.5)
def run(): import libtbx.utils libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser().option( None, "--normal_eqns_solving_method", default='naive').option(None, "--fix_random_seeds", action='store_true', default='naive')).process(args=sys.argv[1:]) opts = command_line.options if opts.fix_random_seeds: import random random.seed(1) flex.set_random_seed(1) gradient_threshold = 1e-8 step_threshold = 1e-8 if opts.normal_eqns_solving_method == 'naive': m = lambda eqns: normal_eqns_solving.naive_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=step_threshold) elif opts.normal_eqns_solving_method == 'levenberg-marquardt': m = lambda eqns: normal_eqns_solving.levenberg_marquardt_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=gradient_threshold, tau=1e-7) else: raise RuntimeError("Unknown method %s" % opts.normal_eqns_solving_method) for t in [ saturated_test_case(m), sucrose_test_case(m), symmetry_equivalent_test_case(m), fpfdp_test_case(m), constrained_fpfdp_test_case(m), scalar_scaled_adp_test_case(m), ]: t.run()
def run(args): if (1): random.seed(0) flex.set_random_seed(0) out = null_out() remaining_args = [] for arg in args: if (arg == "--verbose"): out = sys.stdout else: remaining_args.append(arg) test_cases = test_cases_tardy_pdb.select_test_cases( tags_or_indices=remaining_args) for test_case in test_cases: print >> out, "test case %d: %s" % (test_case.index, test_case.tag) minimized = [] for use_geo in [False, True]: minimized.append(exercise_lbfgs(test_case, use_geo=use_geo, out=out)) m0, m1 = minimized assert m0.real_space_target_weight == 1 assert m1.real_space_target_weight == 1 assert m1.f_final < m0.f_start * 0.98 print format_cpu_times()
def exercise(flags, space_group_info): if not flags.repeats: flags.repeats = 1 if not flags.algo: flags.algo = "weak_reflection_improved" if not flags.on: flags.on = "E" if flags.fix_seed: random.seed(1) flex.set_random_seed(1) n = len(space_group_info.group()) print space_group_info.type().hall_symbol(), if not flags.high_symmetry and n > 24: print " [ skipped ]" if flags.Verbose: print return else: print n_C = 12 // n or 1 n_O = 6 // n n_N = 3 // n if flags.Verbose: print "unit cell content: C%i O%i N%i" % (n_C * n, n_O * n, n_N * n) print "asu content: C%i O%i N%i" % (n_C, n_O, n_N) print "on %s's with %s" % (flags.on, flags.algo) flipping_type = eval("charge_flipping.%s_iterator" % flags.algo) for i in xrange(int(flags.repeats)): randomly_exercise( flipping_type=flipping_type, space_group_info=space_group_info, elements=["C"] * n_C + ["O"] * n_O + ["N"] * n_N, anomalous_flag=False, d_min=0.8, verbose=flags.Verbose, amplitude_type=flags.on, ) if flags.Verbose: print
def run(args): if (1): random.seed(0) flex.set_random_seed(0) out = null_out() remaining_args = [] for arg in args: if (arg == "--verbose"): out = sys.stdout else: remaining_args.append(arg) test_cases = test_cases_tardy_pdb.select_test_cases( tags_or_indices=remaining_args) for test_case in test_cases: print("test case %d: %s" % (test_case.index, test_case.tag), file=out) minimized = [] for use_geo in [False, True]: minimized.append( exercise_lbfgs(test_case, use_geo=use_geo, out=out)) m0, m1 = minimized assert m0.real_space_target_weight == 1 assert m1.real_space_target_weight == 1 assert m1.f_final < m0.f_start * 0.98 print(format_cpu_times())
def exercise_top_largest_f_obs_f_model_differences(threshold_percent=10, symbol = "C 2"): random.seed(0) flex.set_random_seed(0) x = random_structure.xray_structure( space_group_info = sgtbx.space_group_info(symbol=symbol), elements =(("O","N","C")*5+("H",)*10), volume_per_atom = 200, min_distance = 1.5, general_positions_only = True, random_u_iso = True, random_occupancy = False) f_obs = abs(x.structure_factors(d_min = 1.5, algorithm="fft").f_calc()) x.shake_sites_in_place(mean_distance=1) fmodel = mmtbx.f_model.manager( xray_structure = x, f_obs = f_obs) fmodel.update_all_scales(update_f_part1=False) v, d = fmodel.top_largest_f_obs_f_model_differences( threshold_percent=threshold_percent) n = (d>v).count(True)*100./d.size() assert approx_equal(threshold_percent, n, 0.5)
def run(): import libtbx.utils libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser() .option(None, "--normal_eqns_solving_method", default='naive') .option(None, "--fix_random_seeds", action='store_true', default='naive') ).process(args=sys.argv[1:]) opts = command_line.options if opts.fix_random_seeds: import random random.seed(1) flex.set_random_seed(1) gradient_threshold=1e-8 step_threshold=1e-8 if opts.normal_eqns_solving_method == 'naive': m = lambda eqns: normal_eqns_solving.naive_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=step_threshold) elif opts.normal_eqns_solving_method == 'levenberg-marquardt': m = lambda eqns: normal_eqns_solving.levenberg_marquardt_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=gradient_threshold, tau=1e-7) else: raise RuntimeError("Unknown method %s" % opts.normal_eqns_solving_method) for t in [ saturated_test_case(m), sucrose_test_case(m), symmetry_equivalent_test_case(m), ]: t.run()
def run(args): master_phil = libtbx.phil.parse(""" d_min = 0.5 .type = float constrained_refinement = True .type = bool random_seed = 1 .type = int shake_sites_rmsd = 0.5 .type = float shake_adp_spread = 20 .type = float grad_site=True .type = bool grad_u_iso=False .type = bool grad_u_aniso=False .type = bool grad_occupancy=False .type = bool grad_fp_fdp=False .type = bool lbfgs_m = 5 .type = int lbfgs_max_iterations = 1000 .type = int verbose = 0 .type = int """) argument_interpreter = master_phil.command_line_argument_interpreter() phil_objects = [] remaining_args = [] for arg in args: if (arg.find("=") >= 0): phil_objects.append(argument_interpreter.process(arg=arg)) else: remaining_args.append(arg) work_phil = master_phil.fetch(sources=phil_objects) work_phil.show() params = work_phil.extract() if params.random_seed is not None: import scitbx.random import random scitbx.random.set_random_seed(params.random_seed) flex.set_random_seed(params.random_seed) random.seed(params.random_seed) if len(remaining_args): assert len(remaining_args) == 1 file_path = remaining_args[0] root, ext = os.path.splitext(file_path) if ext == ".cif": xs_dict = xray.structure.from_cif(file_path=file_path) assert len( xs_dict) == 1, "CIF should contain only one xray structure" xs = xs_dict.values()[0] xs.show_summary().show_scatterers() print constraints_list = None t_celsius = 20 else: raise RuntimeError("Only CIF format currently supported!") else: test_case = tst_constrained_structure.sucrose_test_case(None) t_celsius = test_case.t_celsius xs = test_case.xray_structure constraints_list = test_case.constraints #from cctbx import adptbx #for sc in xs.scatterers(): #if sc.flags.use_u_aniso(): #sc.u_iso = adptbx.u_star_as_u_iso(xs.unit_cell(), sc.u_star) #sc.set_use_u_iso_only() if not params.constrained_refinement: constraints_list = [] exercise_constrained_lbfgs( xray_structure=xs, constraints_list=constraints_list, t_celsius=t_celsius, d_min=params.d_min, shake_sites_rmsd=params.shake_sites_rmsd, shake_u_iso_spread=params.shake_adp_spread, grad_site=params.grad_site, grad_u_iso=params.grad_u_iso, grad_u_aniso=params.grad_u_aniso, grad_occupancy=params.grad_occupancy, grad_fp_fdp=params.grad_fp_fdp, lbfgs_m=params.lbfgs_m, lbfgs_max_iterations=params.lbfgs_max_iterations, verbose=params.verbose)
def exercise_cif_from_cctbx(): quartz = xray.structure( crystal_symmetry=crystal.symmetry( (5.01,5.01,5.47,90,90,120), "P6222"), scatterers=flex.xray_scatterer([ xray.scatterer("Si", (1/2.,1/2.,1/3.)), xray.scatterer("O", (0.197,-0.197,0.83333))])) for sc in quartz.scatterers(): sc.flags.set_grad_site(True) s = StringIO() loop = geometry.distances_as_cif_loop( quartz.pair_asu_table(distance_cutoff=2), site_labels=quartz.scatterers().extract_labels(), sites_frac=quartz.sites_frac()).loop print >> s, loop assert not show_diff(s.getvalue(), """\ loop_ _geom_bond_atom_site_label_1 _geom_bond_atom_site_label_2 _geom_bond_distance _geom_bond_site_symmetry_2 Si O 1.6160 4_554 Si O 1.6160 2_554 Si O 1.6160 3_664 Si O 1.6160 5_664 """) s = StringIO() loop = geometry.angles_as_cif_loop( quartz.pair_asu_table(distance_cutoff=2), site_labels=quartz.scatterers().extract_labels(), sites_frac=quartz.sites_frac()).loop print >> s, loop assert not show_diff(s.getvalue(), """\ loop_ _geom_angle_atom_site_label_1 _geom_angle_atom_site_label_2 _geom_angle_atom_site_label_3 _geom_angle _geom_angle_site_symmetry_1 _geom_angle_site_symmetry_3 O Si O 101.3 2_554 4_554 O Si O 111.3 3_664 4_554 O Si O 116.1 3_664 2_554 O Si O 116.1 5_664 4_554 O Si O 111.3 5_664 2_554 O Si O 101.3 5_664 3_664 Si O Si 146.9 3 5 """) # with a covariance matrix flex.set_random_seed(1) vcv_matrix = matrix.diag( flex.random_double(size=quartz.n_parameters(), factor=1e-5))\ .as_flex_double_matrix().matrix_symmetric_as_packed_u() s = StringIO() loop = geometry.distances_as_cif_loop( quartz.pair_asu_table(distance_cutoff=2), site_labels=quartz.scatterers().extract_labels(), sites_frac=quartz.sites_frac(), covariance_matrix=vcv_matrix, parameter_map=quartz.parameter_map()).loop print >> s, loop assert not show_diff(s.getvalue(), """\ loop_ _geom_bond_atom_site_label_1 _geom_bond_atom_site_label_2 _geom_bond_distance _geom_bond_site_symmetry_2 Si O 1.616(14) 4_554 Si O 1.616(12) 2_554 Si O 1.616(14) 3_664 Si O 1.616(12) 5_664 """) s = StringIO() loop = geometry.angles_as_cif_loop( quartz.pair_asu_table(distance_cutoff=2), site_labels=quartz.scatterers().extract_labels(), sites_frac=quartz.sites_frac(), covariance_matrix=vcv_matrix, parameter_map=quartz.parameter_map()).loop print >> s, loop assert not show_diff(s.getvalue(), """\ loop_ _geom_angle_atom_site_label_1 _geom_angle_atom_site_label_2 _geom_angle_atom_site_label_3 _geom_angle _geom_angle_site_symmetry_1 _geom_angle_site_symmetry_3 O Si O 101.3(8) 2_554 4_554 O Si O 111.3(10) 3_664 4_554 O Si O 116.1(9) 3_664 2_554 O Si O 116.1(9) 5_664 4_554 O Si O 111.3(10) 5_664 2_554 O Si O 101.3(8) 5_664 3_664 Si O Si 146.9(9) 3 5 """) cell_vcv = flex.pow2(matrix.diag(flex.random_double(size=6,factor=1e-1))\ .as_flex_double_matrix().matrix_symmetric_as_packed_u()) s = StringIO() loop = geometry.distances_as_cif_loop( quartz.pair_asu_table(distance_cutoff=2), site_labels=quartz.scatterers().extract_labels(), sites_frac=quartz.sites_frac(), covariance_matrix=vcv_matrix, cell_covariance_matrix=cell_vcv, parameter_map=quartz.parameter_map()).loop print >> s, loop assert not show_diff(s.getvalue(), """\ loop_ _geom_bond_atom_site_label_1 _geom_bond_atom_site_label_2 _geom_bond_distance _geom_bond_site_symmetry_2 Si O 1.616(15) 4_554 Si O 1.616(19) 2_554 Si O 1.616(15) 3_664 Si O 1.616(19) 5_664 """)
def run(args): from dials.util import log usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name parser = OptionParser(usage=usage, phil=phil_scope, read_datablocks=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=False) datablocks = flatten_datablocks(params.input.datablock) reflections = flatten_reflections(params.input.reflections) if len(datablocks) == 0 or len(reflections) == 0: parser.print_help() exit(0) # Configure the logging log.config(info=params.output.log, debug=params.output.debug_log) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) if params.seed is not None: import random flex.set_random_seed(params.seed) random.seed(params.seed) imagesets = [] for datablock in datablocks: imagesets.extend(datablock.extract_imagesets()) assert len(imagesets) > 0 assert len(reflections) == len(imagesets) if params.scan_range is not None and len(params.scan_range) > 0: reflections = [ filter_reflections_by_scan_range(refl, params.scan_range) for refl in reflections ] dps_params = dps_phil_scope.extract() # for development, we want an exhaustive plot of beam probability map: dps_params.indexing.plot_search_scope = params.plot_search_scope dps_params.indexing.mm_search_scope = params.mm_search_scope for i in range(params.n_macro_cycles): if params.n_macro_cycles > 1: logger.info('Starting macro cycle %i' % (i + 1)) new_detector, new_beam = discover_better_experimental_model( imagesets, reflections, params, dps_params, nproc=params.nproc, wide_search_binning=params.wide_search_binning) for imageset in imagesets: imageset.set_detector(new_detector) imageset.set_beam(new_beam) logger.info('') from dxtbx.serialize import dump logger.info("Saving optimized datablock to %s" % params.output.datablock) dump.datablock(datablock, params.output.datablock)
def try_den_weight_cartesian(self, grid_pair): local_seed = int(self.random_seed+grid_pair[1]) flex.set_random_seed(value=local_seed) random.seed(local_seed) self.fmodels.fmodel_xray().xray_structure.replace_scatterers( self.save_scatterers_local.deep_copy()) self.fmodels.update_xray_structure( xray_structure = self.fmodels.fmodel_xray().xray_structure, update_f_calc = True) utils.assert_xray_structures_equal( x1 = self.fmodels.fmodel_xray().xray_structure, x2 = self.model.get_xray_structure()) gamma_local = grid_pair[0] weight_local = grid_pair[1] self.model.restraints_manager.geometry.\ den_manager.gamma = gamma_local self.model.restraints_manager.geometry.\ den_manager.weight = weight_local cycle = 0 self.model.restraints_manager.geometry.\ den_manager.current_cycle = cycle+1 num_den_cycles = self.model.restraints_manager.geometry.\ den_manager.num_cycles if self.params.den.optimize and \ self.nproc != Auto and \ self.nproc > 1: local_log = sys.stdout elif self.params.den.optimize and \ self.nproc == 1: if self.verbose: local_log = self.log else: local_log = StringIO() else: local_log = self.log print >> self.log, " ...trying gamma %f, weight %f" % ( gamma_local, weight_local) while cycle < num_den_cycles: print >> local_log, "DEN cycle %s" % (cycle+1) r_free = self.fmodels.fmodel_xray().r_free() print >> local_log, "rfree at start of SA cycle: %f" % r_free simulated_annealing.manager( params = self.params.simulated_annealing, target_weights = self.target_weights, macro_cycle = self.macro_cycle, h_params = self.params.hydrogens, fmodels = self.fmodels, model = self.model, all_params = self.params, out = local_log) if self.params.den.bulk_solvent_and_scale: self.bulk_solvent_and_scale(log=local_log) if self.params.den.refine_adp: self.adp_refinement(log=local_log) self.model.torsion_ncs_restraints_update(log=local_log) cycle += 1 self.model.restraints_manager.geometry.\ den_manager.current_cycle += 1 r_free = self.fmodels.fmodel_xray().r_free() print >> local_log, "rfree at end of SA cycle: %f" % r_free r_free = self.fmodels.fmodel_xray().r_free() step_xray_structure = self.fmodels.fmodel_xray().\ xray_structure.deep_copy_scatterers().scatterers() step_eq_distances = self.model.restraints_manager.geometry.\ den_manager.get_current_eq_distances() return (gamma_local, weight_local, r_free, step_xray_structure, step_eq_distances)
def run( args, command_name="phenix.reflection_file_converter", simply_return_all_miller_arrays=False): command_line = (option_parser( usage="%s [options] reflection_file ..." % command_name, description="Example: %s w1.sca --mtz ." % command_name) .enable_symmetry_comprehensive() .option(None, "--weak_symmetry", action="store_true", default=False, help="symmetry on command line is weaker than symmetry found in files") .enable_resolutions() .option(None, "--label", action="store", type="string", help="Substring of reflection data label or number", metavar="STRING") .option(None, "--non_anomalous", action="store_true", default=False, help="Averages Bijvoet mates to obtain a non-anomalous array") .option(None, "--r_free_label", action="store", type="string", help="Substring of reflection data label or number", metavar="STRING") .option(None, "--r_free_test_flag_value", action="store", type="int", help="Value in R-free array indicating assignment to free set.", metavar="FLOAT") .option(None, "--generate_r_free_flags", action="store_true", default=False, help="Generates a new array of random R-free flags" " (MTZ and CNS output only).") .option(None, "--use_lattice_symmetry_in_r_free_flag_generation", dest="use_lattice_symmetry_in_r_free_flag_generation", action="store_true", default=True, help="group twin/pseudo symmetry related reflections together" " in r-free set (this is the default).") .option(None, "--no_lattice_symmetry_in_r_free_flag_generation", dest="use_lattice_symmetry_in_r_free_flag_generation", action="store_false", help="opposite of --use-lattice-symmetry-in-r-free-flag-generation") .option(None, "--r_free_flags_fraction", action="store", default=0.10, type="float", help="Target fraction free/work reflections (default: 0.10).", metavar="FLOAT") .option(None, "--r_free_flags_max_free", action="store", default=2000, type="int", help="Maximum number of free reflections (default: 2000).", metavar="INT") .option(None, "--r_free_flags_format", choices=("cns", "ccp4", "shelx"), default="cns", help="Convention for generating R-free flags", metavar="cns|ccp4") .option(None, "--output_r_free_label", action="store", type="string", help="Label for newly generated R-free flags (defaults to R-free-flags)", default="R-free-flags", metavar="STRING") .option(None, "--random_seed", action="store", type="int", help="Seed for random number generator (affects generation of" " R-free flags).", metavar="INT") .option(None, "--change_of_basis", action="store", type="string", help="Change-of-basis operator: h,k,l or x,y,z" " or to_reference_setting, to_primitive_setting, to_niggli_cell," " to_inverse_hand", metavar="STRING") .option(None, "--eliminate_invalid_indices", action="store_true", default=False, help="Remove indices which are invalid given the change of basis desired") .option(None, "--expand_to_p1", action="store_true", default=False, help="Generates all symmetrically equivalent reflections." " The space group symmetry is reset to P1." " May be used in combination with --change_to_space_group to" " lower the symmetry.") .option(None, "--change_to_space_group", action="store", type="string", help="Changes the space group and merges equivalent reflections" " if necessary", metavar="SYMBOL|NUMBER") .option(None, "--write_mtz_amplitudes", action="store_true", default=False, help="Converts intensities to amplitudes before writing MTZ format;" " requires --mtz_root_label") .option(None, "--write_mtz_intensities", action="store_true", default=False, help="Converts amplitudes to intensities before writing MTZ format;" " requires --mtz_root_label") .option(None,"--remove_negatives", action="store_true", default=False, help="Remove negative intensities or amplitudes from the data set" ) .option(None,"--massage_intensities", action="store_true", default=False, help="'Treat' negative intensities to get a positive amplitude." " |Fnew| = sqrt((Io+sqrt(Io**2 +2sigma**2))/2.0). Requires" " intensities as input and the flags --mtz," " --write_mtz_amplitudes and --mtz_root_label.") .option(None, "--scale_max", action="store", type="float", help="Scales data such that the maximum is equal to the given value", metavar="FLOAT") .option(None, "--scale_factor", action="store", type="float", help="Multiplies data with the given factor", metavar="FLOAT") .option(None, "--sca", action="store", type="string", help= "write data to Scalepack FILE ('--sca .' copies name of input file)", metavar="FILE") .option(None, "--mtz", action="store", type="string", help="write data to MTZ FILE ('--mtz .' copies name of input file)", metavar="FILE") .option(None, "--mtz_root_label", action="store", type="string", help="Root label for MTZ file (e.g. Fobs)", metavar="STRING") .option(None, "--cns", action="store", type="string", help="write data to CNS FILE ('--cns .' copies name of input file)", metavar="FILE") .option(None, "--shelx", action="store", type="string", help="write data to SHELX FILE ('--shelx .' copies name of input file)", metavar="FILE") ).process(args=args) co = command_line.options if (co.random_seed is not None): random.seed(co.random_seed) flex.set_random_seed(value=co.random_seed) if ( co.write_mtz_amplitudes and co.write_mtz_intensities): print print "--write_mtz_amplitudes and --write_mtz_intensities" \ " are mutually exclusive." print return None if ( co.write_mtz_amplitudes or co.write_mtz_intensities): if (co.mtz_root_label is None): print print "--write_mtz_amplitudes and --write_mtz_intensities" \ " require --mtz_root_label." print return None if ( co.scale_max is not None and co.scale_factor is not None): print print "--scale_max and --scale_factor are mutually exclusive." print return None if (len(command_line.args) == 0): command_line.parser.show_help() return None all_miller_arrays = reflection_file_reader.collect_arrays( file_names=command_line.args, crystal_symmetry=None, force_symmetry=False, merge_equivalents=False, discard_arrays=False, verbose=1) if (simply_return_all_miller_arrays): return all_miller_arrays if (len(all_miller_arrays) == 0): print print "No reflection data found in input file%s." % ( plural_s(len(command_line.args))[1]) print return None label_table = reflection_file_utils.label_table( miller_arrays=all_miller_arrays) selected_array = label_table.select_array( label=co.label, command_line_switch="--label") if (selected_array is None): return None r_free_flags = None r_free_info = None if (co.r_free_label is not None): r_free_flags = label_table.match_data_label( label=co.r_free_label, command_line_switch="--r_free_label") if (r_free_flags is None): return None r_free_info = str(r_free_flags.info()) if (not r_free_flags.is_bool_array()): test_flag_value = reflection_file_utils.get_r_free_flags_scores( miller_arrays=[r_free_flags], test_flag_value=co.r_free_test_flag_value).test_flag_values[0] if (test_flag_value is None): if (co.r_free_test_flag_value is None): raise Sorry( "Cannot automatically determine r_free_test_flag_value." " Please use --r_free_test_flag_value to specify a value.") else: raise Sorry("Invalid --r_free_test_flag_value.") r_free_flags = r_free_flags.customized_copy( data=(r_free_flags.data() == test_flag_value)) print "Selected data:" print " ", selected_array.info() print " Observation type:", selected_array.observation_type() print if (r_free_info is not None): print "R-free flags:" print " ", r_free_info print processed_array = selected_array.customized_copy( crystal_symmetry=selected_array.join_symmetry( other_symmetry=command_line.symmetry, force=not co.weak_symmetry)).set_observation_type( selected_array.observation_type()) if (r_free_flags is not None): r_free_flags = r_free_flags.customized_copy( crystal_symmetry=processed_array) print "Input crystal symmetry:" crystal.symmetry.show_summary(processed_array, prefix=" ") print if (processed_array.unit_cell() is None): command_line.parser.show_help() print "Unit cell parameters unknown. Please use --symmetry or --unit_cell." print return None if (processed_array.space_group_info() is None): command_line.parser.show_help() print "Space group unknown. Please use --symmetry or --space_group." print return None if (r_free_flags is not None): r_free_flags = r_free_flags.customized_copy( crystal_symmetry=processed_array) if (co.change_of_basis is not None): processed_array, cb_op = processed_array.apply_change_of_basis( change_of_basis=co.change_of_basis, eliminate_invalid_indices=co.eliminate_invalid_indices) if (r_free_flags is not None): r_free_flags = r_free_flags.change_basis(cb_op=cb_op) if (not processed_array.is_unique_set_under_symmetry()): print "Merging symmetry-equivalent values:" merged = processed_array.merge_equivalents() merged.show_summary(prefix=" ") print processed_array = merged.array() del merged processed_array.show_comprehensive_summary(prefix=" ") print if (r_free_flags is not None and not r_free_flags.is_unique_set_under_symmetry()): print "Merging symmetry-equivalent R-free flags:" merged = r_free_flags.merge_equivalents() merged.show_summary(prefix=" ") print r_free_flags = merged.array() del merged r_free_flags.show_comprehensive_summary(prefix=" ") print if (co.expand_to_p1): print "Expanding symmetry and resetting space group to P1:" if (r_free_flags is not None): raise Sorry( "--expand_to_p1 not supported for arrays of R-free flags.") processed_array = processed_array.expand_to_p1() processed_array.show_comprehensive_summary(prefix=" ") print if (co.change_to_space_group is not None): if (r_free_flags is not None): raise Sorry( "--change_to_space_group not supported for arrays of R-free flags.") new_space_group_info = sgtbx.space_group_info( symbol=co.change_to_space_group) print "Change to space group:", new_space_group_info new_crystal_symmetry = crystal.symmetry( unit_cell=processed_array.unit_cell(), space_group_info=new_space_group_info, assert_is_compatible_unit_cell=False) if (not new_crystal_symmetry.unit_cell() .is_similar_to(processed_array.unit_cell())): print " *************" print " W A R N I N G" print " *************" print " Unit cell parameters adapted to new space group symmetry are" print " significantly different from input unit cell parameters:" print " Input unit cell parameters:", \ processed_array.unit_cell() print " Adapted unit cell parameters:", \ new_crystal_symmetry.unit_cell() processed_array = processed_array.customized_copy( crystal_symmetry=new_crystal_symmetry) print if (not processed_array.is_unique_set_under_symmetry()): print " Merging values symmetry-equivalent under new symmetry:" merged = processed_array.merge_equivalents() merged.show_summary(prefix=" ") print processed_array = merged.array() del merged processed_array.show_comprehensive_summary(prefix=" ") print if (processed_array.anomalous_flag() and co.non_anomalous): print "Converting data array from anomalous to non-anomalous." if (not processed_array.is_xray_intensity_array()): processed_array = processed_array.average_bijvoet_mates() else: processed_array = processed_array.f_sq_as_f() processed_array = processed_array.average_bijvoet_mates() processed_array = processed_array.f_as_f_sq() processed_array.set_observation_type_xray_intensity() if (r_free_flags is not None and r_free_flags.anomalous_flag() and co.non_anomalous): print "Converting R-free flags from anomalous to non-anomalous." r_free_flags = r_free_flags.average_bijvoet_mates() d_max = co.low_resolution d_min = co.resolution if (d_max is not None or d_min is not None): if (d_max is not None): print "Applying low resolution cutoff: d_max=%.6g" % d_max if (d_min is not None): print "Applying high resolution cutoff: d_min=%.6g" % d_min processed_array = processed_array.resolution_filter( d_max=d_max, d_min=d_min) print "Number of reflections:", processed_array.indices().size() print if (co.scale_max is not None): print "Scaling data such that the maximum value is: %.6g" % co.scale_max processed_array = processed_array.apply_scaling(target_max=co.scale_max) print if (co.scale_factor is not None): print "Multiplying data with the factor: %.6g" % co.scale_factor processed_array = processed_array.apply_scaling(factor=co.scale_factor) print if (([co.remove_negatives, co.massage_intensities]).count(True) == 2): raise Sorry( "It is not possible to use --remove_negatives and" " --massage_intensities at the same time.") if (co.remove_negatives): if processed_array.is_real_array(): print "Removing negatives items" processed_array = processed_array.select( processed_array.data() > 0) if processed_array.sigmas() is not None: processed_array = processed_array.select( processed_array.sigmas() > 0) else: raise Sorry("--remove_negatives not applicable to complex data arrays.") if (co.massage_intensities): if processed_array.is_real_array(): if processed_array.is_xray_intensity_array(): if (co.mtz is not None): if (co.write_mtz_amplitudes): print "The supplied intensities will be used to estimate" print " amplitudes in the following way: " print " Fobs = Sqrt[ (Iobs + Sqrt(Iobs**2 + 2sigmaIobs**2))/2 ]" print " Sigmas are estimated in a similar manner." print processed_array = processed_array.enforce_positive_amplitudes() else: raise Sorry( "--write_mtz_amplitudes has to be specified when using" " --massage_intensities") else: raise Sorry("--mtz has to be used when using --massage_intensities") else: raise Sorry( "Intensities must be supplied when using the option" " --massage_intensities") else: raise Sorry( "--massage_intensities not applicable to complex data arrays.") if (not co.generate_r_free_flags): if (r_free_flags is None): r_free_info = [] else: if (r_free_flags.anomalous_flag() != processed_array.anomalous_flag()): if (processed_array.anomalous_flag()): is_not = ("", " not") else: is_not = (" not", "") raise Sorry( "The data array is%s anomalous but the R-free array is%s.\n" % is_not + " Please try --non_anomalous.") r_free_info = ["R-free flags source: " + r_free_info] if (not r_free_flags.indices().all_eq(processed_array.indices())): processed_array = processed_array.map_to_asu() r_free_flags = r_free_flags.map_to_asu().common_set(processed_array) n_missing_r_free_flags = processed_array.indices().size() \ - r_free_flags.indices().size() if (n_missing_r_free_flags != 0): raise Sorry("R-free flags not compatible with data array:" " missing flag for %d reflections selected for output." % n_missing_r_free_flags) else: if (r_free_flags is not None): raise Sorry( "--r_free_label and --generate_r_free_flags are mutually exclusive.") print "Generating a new array of R-free flags:" r_free_flags = processed_array.generate_r_free_flags( fraction=co.r_free_flags_fraction, max_free=co.r_free_flags_max_free, use_lattice_symmetry=co.use_lattice_symmetry_in_r_free_flag_generation, format=co.r_free_flags_format) test_flag_value = True if (co.r_free_flags_format == "ccp4") : test_flag_value = 0 elif (co.r_free_flags_format == "shelx") : test_flag_value = -1 r_free_as_bool = r_free_flags.customized_copy( data=r_free_flags.data()==test_flag_value) r_free_info = [ "R-free flags generated by %s:" % command_name] r_free_info.append(" "+date_and_time()) r_free_info.append(" fraction: %.6g" % co.r_free_flags_fraction) r_free_info.append(" max_free: %s" % str(co.r_free_flags_max_free)) r_free_info.append(" size of work set: %d" % r_free_as_bool.data().count(False)) r_free_info.append(" size of free set: %d" % r_free_as_bool.data().count(True)) r_free_info_str = StringIO() r_free_as_bool.show_r_free_flags_info(prefix=" ", out=r_free_info_str) if (co.r_free_flags_format == "ccp4") : r_free_info.append(" convention: CCP4 (test=0, work=1-%d)" % flex.max(r_free_flags.data())) elif (co.r_free_flags_format == "shelx") : r_free_info.append(" convention: SHELXL (test=-1, work=1)") else : r_free_info.append(" convention: CNS/X-PLOR (test=1, work=0)") print "\n".join(r_free_info[2:4]) print r_free_info[-1] print r_free_info_str.getvalue() print n_output_files = 0 if (co.sca is not None): if (co.generate_r_free_flags): raise Sorry("Cannot write R-free flags to Scalepack file.") file_name = reflection_file_utils.construct_output_file_name( input_file_names=[selected_array.info().source], user_file_name=co.sca, file_type_label="Scalepack", file_extension="sca") print "Writing Scalepack file:", file_name iotbx.scalepack.merge.write( file_name=file_name, miller_array=processed_array) n_output_files += 1 print if (co.mtz is not None): file_name = reflection_file_utils.construct_output_file_name( input_file_names=[selected_array.info().source], user_file_name=co.mtz, file_type_label="MTZ", file_extension="mtz") print "Writing MTZ file:", file_name mtz_history_buffer = flex.std_string() mtz_history_buffer.append(date_and_time()) mtz_history_buffer.append("> program: %s" % command_name) mtz_history_buffer.append("> input file name: %s" % os.path.basename(selected_array.info().source)) mtz_history_buffer.append("> input directory: %s" % os.path.dirname(os.path.abspath(selected_array.info().source))) mtz_history_buffer.append("> input labels: %s" % selected_array.info().label_string()) mtz_output_array = processed_array if (co.write_mtz_amplitudes): if (not mtz_output_array.is_xray_amplitude_array()): print " Converting intensities to amplitudes." mtz_output_array = mtz_output_array.f_sq_as_f() mtz_history_buffer.append("> Intensities converted to amplitudes.") elif (co.write_mtz_intensities): if (not mtz_output_array.is_xray_intensity_array()): print " Converting amplitudes to intensities." mtz_output_array = mtz_output_array.f_as_f_sq() mtz_history_buffer.append("> Amplitudes converted to intensities.") column_root_label = co.mtz_root_label if (column_root_label is None): # XXX 2013-03-29: preserve original root label by default # XXX 2014-12-16: skip trailing "(+)" in root_label if anomalous column_root_label = selected_array.info().labels[0] column_root_label=remove_anomalous_suffix_if_necessary( miller_array=selected_array, column_root_label=column_root_label) mtz_dataset = mtz_output_array.as_mtz_dataset( column_root_label=column_root_label) del mtz_output_array if (r_free_flags is not None): mtz_dataset.add_miller_array( miller_array=r_free_flags, column_root_label=co.output_r_free_label) for line in r_free_info: mtz_history_buffer.append("> " + line) mtz_history_buffer.append("> output file name: %s" % os.path.basename(file_name)) mtz_history_buffer.append("> output directory: %s" % os.path.dirname(os.path.abspath(file_name))) mtz_object = mtz_dataset.mtz_object() mtz_object.add_history(mtz_history_buffer) mtz_object.write(file_name=file_name) n_output_files += 1 print if (co.cns is not None): file_name = reflection_file_utils.construct_output_file_name( input_file_names=[selected_array.info().source], user_file_name=co.cns, file_type_label="CNS", file_extension="cns") print "Writing CNS file:", file_name processed_array.export_as_cns_hkl( file_object=open(file_name, "w"), file_name=file_name, info=["source of data: "+str(selected_array.info())] + r_free_info, r_free_flags=r_free_flags) n_output_files += 1 print if (co.shelx is not None): if (co.generate_r_free_flags): raise Sorry("Cannot write R-free flags to SHELX file.") file_name = reflection_file_utils.construct_output_file_name( input_file_names=[selected_array.info().source], user_file_name=co.shelx, file_type_label="SHELX", file_extension="shelx") print "Writing SHELX file:", file_name processed_array.as_amplitude_array().export_as_shelx_hklf( open(file_name, "w")) n_output_files += 1 print if (n_output_files == 0): command_line.parser.show_help() print "Please specify at least one output file format,", print "e.g. --mtz, --sca, etc." print return None return processed_array
def run(args): import cctbx.omz.dev import iotbx.cif import cctbx.xray # import random random.seed(0) from cctbx.array_family import flex flex.set_random_seed(0) # master_phil = cctbx.omz.dev.get_master_phil( iteration_limit=100, additional_phil_string="""\ remove_hydrogen = True .type = bool f_obs_is_f_calc = True .type = bool reset_u_iso = None .type = float """) argument_interpreter = master_phil.command_line_argument_interpreter() phil_objects = [] remaining_args = [] for arg in args: if (arg.find("=") >= 0): phil_objects.append(argument_interpreter.process(arg=arg)) else: remaining_args.append(arg) work_phil = master_phil.fetch(sources=phil_objects) work_phil.show() params = work_phil.extract() print # assert len(remaining_args) == 2, "refl_cif.hkl model.cif" refl_file = remaining_args[0] model_file = remaining_args[1] refl_cif = iotbx.cif.reader(file_path=refl_file) model_cif = iotbx.cif.reader(file_path=model_file) # mgr = extract_from_cif_files( report_id="cif_refine", refl_file=refl_file, refl_cif=refl_cif, model_file=model_file, model_cif=model_cif) # structure_ideal = mgr.xray_structure.deep_copy_scatterers() structure_ideal.convert_to_isotropic() if (params.remove_hydrogen): sel = mgr.non_hydrogen_selection print "Removing hydrogen atoms:", sel.count(False) structure_ideal = structure_ideal.select(selection=sel) print # c_obs = mgr.c_obs if (params.f_obs_is_f_calc): c_obs = c_obs.structure_factors_from_scatterers( xray_structure=structure_ideal, algorithm="direct", cos_sin_table=False).f_calc().amplitudes() \ .set_observation_type_xray_amplitude() # assert c_obs.is_xray_intensity_array() or c_obs.is_xray_amplitude_array() if (c_obs.is_xray_intensity_array()): i_obs = c_obs f_obs = c_obs.f_sq_as_f(algorithm="xtal_3_7") else: f_obs = c_obs i_obs = c_obs.f_as_f_sq(algorithm="shelxl") # structure_shake = structure_ideal.deep_copy_scatterers() structure_shake.shake_sites_in_place(rms_difference=params.shake_sites_rmsd) if (params.reset_u_iso is None): structure_shake.shake_adp(spread=params.shake_adp_spread) else: structure_shake.set_u_iso(value=params.reset_u_iso) # cctbx.omz.dev.run_refinement( structure_ideal=structure_ideal, structure_shake=structure_shake, params=params, f_obs=f_obs, i_obs=i_obs)
def try_den_weight_torsion(self, grid_pair): #backup_k_rep = self.params.tardy.\ # prolsq_repulsion_function_changes.k_rep local_seed = int(self.random_seed+grid_pair[1]) flex.set_random_seed(value=local_seed) random.seed(local_seed) self.fmodels.fmodel_xray().xray_structure.replace_scatterers( self.save_scatterers_local.deep_copy()) self.fmodels.update_xray_structure( xray_structure = self.fmodels.fmodel_xray().xray_structure, update_f_calc = True) utils.assert_xray_structures_equal( x1 = self.fmodels.fmodel_xray().xray_structure, x2 = self.model.get_xray_structure()) gamma_local = grid_pair[0] weight_local = grid_pair[1] self.model.restraints_manager.geometry.\ den_manager.gamma = gamma_local self.model.restraints_manager.geometry.\ den_manager.weight = weight_local cycle = 0 self.model.restraints_manager.geometry.\ den_manager.current_cycle = cycle+1 num_den_cycles = self.model.restraints_manager.geometry.\ den_manager.num_cycles if self.params.den.optimize and \ self.nproc != Auto and \ self.nproc > 1: local_log = sys.stdout elif self.params.den.optimize and \ self.nproc == 1: if self.verbose: local_log = self.log else: local_log = StringIO() else: local_log = self.log print >> self.log, " ...trying gamma %.1f, weight %.1f" % ( gamma_local, weight_local) while cycle < num_den_cycles: #if self.model.restraints_manager.geometry.\ # generic_restraints_manager.den_manager.current_cycle == \ # self.model.restraints_manager.geometry.\ # generic_restraints_manager.den_manager.torsion_mid_point+1: # self.params.tardy.\ # prolsq_repulsion_function_changes.k_rep = 1.0 print >> local_log, "DEN cycle %d" % (cycle+1) #print >> local_log, "Random seed: %d" % flex.get_random_seed() r_free = self.fmodels.fmodel_xray().r_free() print >> local_log, "rfree at start of SA cycle: %.4f" % r_free print >> local_log, "k_rep = %.2f" % \ self.params.tardy.\ prolsq_repulsion_function_changes.k_rep tardy.run( fmodels=self.fmodels, model=self.model, target_weights=self.target_weights, params=self.params.tardy, log=local_log, format_for_phenix_refine=True, call_back_after_step=False) if self.params.den.bulk_solvent_and_scale: self.bulk_solvent_and_scale(log=local_log) self.fmodels.fmodel_xray().xray_structure = self.model.get_xray_structure() if self.params.den.refine_adp: self.adp_refinement(log=local_log) self.model.torsion_ncs_restraints_update(log=local_log) cycle += 1 self.model.restraints_manager.geometry.\ den_manager.current_cycle += 1 r_free = self.fmodels.fmodel_xray().r_free() print >> local_log, "rfree at end of SA cycle: %f" % r_free r_free = self.fmodels.fmodel_xray().r_free() step_xray_structure = self.fmodels.fmodel_xray().\ xray_structure.deep_copy_scatterers().scatterers() step_eq_distances = self.model.restraints_manager.geometry.\ den_manager.get_current_eq_distances() return (gamma_local, weight_local, r_free, step_xray_structure, step_eq_distances)
def exercise_restrained_refinement(options): import random random.seed(1) flex.set_random_seed(1) xs0 = smtbx.development.random_xray_structure( sgtbx.space_group_info('P1'), n_scatterers=options.n_scatterers, elements="random") for sc in xs0.scatterers(): sc.flags.set_grad_site(True) sc0 = xs0.scatterers() uc = xs0.unit_cell() mi = xs0.build_miller_set(anomalous_flag=False, d_min=options.resolution) fo_sq = mi.structure_factors_from_scatterers( xs0, algorithm="direct").f_calc().norm() fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1)) i, j, k, l = random.sample(xrange(options.n_scatterers), 4) bond_proxies = geometry_restraints.shared_bond_simple_proxy() w = 1e9 d_ij = uc.distance(sc0[i].site, sc0[j].site)*0.8 bond_proxies.append(geom.bond_simple_proxy( i_seqs=(i, j), distance_ideal=d_ij, weight=w)) d_jk = uc.distance(sc0[j].site, sc0[k].site)*0.85 bond_proxies.append(geom.bond_simple_proxy( i_seqs=(j, k), distance_ideal=d_jk, weight=w)) d_ki = min(uc.distance(sc0[k].site, sc0[i].site)*0.9, (d_ij + d_jk)*0.8) bond_proxies.append(geom.bond_simple_proxy( i_seqs=(k, i), distance_ideal=d_ki, weight=w)) d_jl = uc.distance(sc0[j].site, sc0[l].site)*0.9 bond_proxies.append(geom.bond_simple_proxy( i_seqs=(j, l), distance_ideal=d_jl, weight=w)) d_lk = min(uc.distance(sc0[l].site, sc0[k].site)*0.8, 0.75*(d_jk + d_jl)) bond_proxies.append(geom.bond_simple_proxy( i_seqs=(l, k), distance_ideal=d_jl, weight=w)) restraints_manager = restraints.manager(bond_proxies=bond_proxies) xs1 = xs0.deep_copy_scatterers() xs1.shake_sites_in_place(rms_difference=0.1) def ls_problem(): xs = xs1.deep_copy_scatterers() reparametrisation = constraints.reparametrisation( structure=xs, constraints=[], connectivity_table=smtbx.utils.connectivity_table(xs), temperature=20) return least_squares.crystallographic_ls( fo_sq.as_xray_observations(), reparametrisation=reparametrisation, restraints_manager=restraints_manager) gradient_threshold, step_threshold = 1e-6, 1e-6 eps = 5e-3 ls = ls_problem() t = wall_clock_time() cycles = normal_eqns_solving.naive_iterations( ls, gradient_threshold=gradient_threshold, step_threshold=step_threshold, track_all=True) if options.verbose: print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed()) sc = ls.xray_structure.scatterers() for p in bond_proxies: d = uc.distance(*[ sc[i_pair].site for i_pair in p.i_seqs ]) assert approx_equal(d, p.distance_ideal, eps) ls = ls_problem() t = wall_clock_time() cycles = normal_eqns_solving.levenberg_marquardt_iterations( ls, gradient_threshold=gradient_threshold, step_threshold=step_threshold, tau=1e-3, track_all=True) if options.verbose: print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed()) sc = ls.xray_structure.scatterers() sc = ls.xray_structure.scatterers() for p in bond_proxies: d = uc.distance(*[ sc[i].site for i in p.i_seqs ]) assert approx_equal(d, p.distance_ideal, eps)
def exercise_hbond_as_cif_loop(): xs = sucrose() for sc in xs.scatterers(): sc.flags.set_grad_site(True) radii = [ covalent_radii.table(elt).radius() for elt in xs.scattering_type_registry().type_index_pairs_as_dict() ] asu_mappings = xs.asu_mappings( buffer_thickness=2*max(radii) + 0.5) pair_asu_table = crystal.pair_asu_table(asu_mappings) pair_asu_table.add_covalent_pairs( xs.scattering_types(), tolerance=0.5) hbonds = [ geometry.hbond(1,5, sgtbx.rt_mx('-X,0.5+Y,2-Z')), geometry.hbond(5,14, sgtbx.rt_mx('-X,-0.5+Y,1-Z')), geometry.hbond(7,10, sgtbx.rt_mx('1+X,+Y,+Z')), geometry.hbond(10,0), geometry.hbond(12,14, sgtbx.rt_mx('-1-X,0.5+Y,1-Z')), geometry.hbond(14,12, sgtbx.rt_mx('-1-X,-0.5+Y,1-Z')), geometry.hbond(16,7) ] loop = geometry.hbonds_as_cif_loop( hbonds, pair_asu_table, xs.scatterers().extract_labels(), sites_frac=xs.sites_frac()).loop s = StringIO() print >> s, loop assert not show_diff(s.getvalue(), """\ loop_ _geom_hbond_atom_site_label_D _geom_hbond_atom_site_label_H _geom_hbond_atom_site_label_A _geom_hbond_distance_DH _geom_hbond_distance_HA _geom_hbond_distance_DA _geom_hbond_angle_DHA _geom_hbond_site_symmetry_A O2 H2 O4 0.8200 2.0636 2.8635 165.0 2_557 O4 H4 O9 0.8200 2.0559 2.8736 174.9 2_546 O5 H5 O7 0.8200 2.0496 2.8589 169.0 1_655 O7 H7 O1 0.8200 2.0573 2.8617 166.8 . O8 H8 O9 0.8200 2.1407 2.8943 152.8 2_456 O9 H9 O8 0.8200 2.1031 2.8943 162.1 2_446 O10 H10 O5 0.8200 2.0167 2.7979 159.1 . """) # with a covariance matrix flex.set_random_seed(1) vcv_matrix = matrix.diag( flex.random_double(size=xs.n_parameters(), factor=1e-5))\ .as_flex_double_matrix().matrix_symmetric_as_packed_u() loop = geometry.hbonds_as_cif_loop( hbonds, pair_asu_table, xs.scatterers().extract_labels(), sites_frac=xs.sites_frac(), covariance_matrix=vcv_matrix, parameter_map=xs.parameter_map()).loop s = StringIO() print >> s, loop assert not show_diff(s.getvalue(), """\ loop_ _geom_hbond_atom_site_label_D _geom_hbond_atom_site_label_H _geom_hbond_atom_site_label_A _geom_hbond_distance_DH _geom_hbond_distance_HA _geom_hbond_distance_DA _geom_hbond_angle_DHA _geom_hbond_site_symmetry_A O2 H2 O4 0.82(3) 2.06(3) 2.86(3) 165.0(18) 2_557 O4 H4 O9 0.82(4) 2.06(4) 2.87(4) 175(2) 2_546 O5 H5 O7 0.82(2) 2.05(2) 2.859(19) 169.0(18) 1_655 O7 H7 O1 0.82(2) 2.06(2) 2.86(2) 167(2) . O8 H8 O9 0.82(3) 2.14(3) 2.89(3) 153(3) 2_456 O9 H9 O8 0.82(3) 2.10(3) 2.89(3) 162(2) 2_446 O10 H10 O5 0.82(3) 2.02(3) 2.80(3) 159(3) . """) cell_vcv = flex.pow2(matrix.diag(flex.random_double(size=6,factor=1e-1))\ .as_flex_double_matrix().matrix_symmetric_as_packed_u()) loop = geometry.hbonds_as_cif_loop( hbonds, pair_asu_table, xs.scatterers().extract_labels(), sites_frac=xs.sites_frac(), covariance_matrix=vcv_matrix, cell_covariance_matrix=cell_vcv, parameter_map=xs.parameter_map()).loop s = StringIO() print >> s, loop assert not show_diff(s.getvalue(), """\ loop_ _geom_hbond_atom_site_label_D _geom_hbond_atom_site_label_H _geom_hbond_atom_site_label_A _geom_hbond_distance_DH _geom_hbond_distance_HA _geom_hbond_distance_DA _geom_hbond_angle_DHA _geom_hbond_site_symmetry_A O2 H2 O4 0.82(3) 2.06(4) 2.86(4) 165.0(18) 2_557 O4 H4 O9 0.82(4) 2.06(4) 2.87(4) 175(2) 2_546 O5 H5 O7 0.82(2) 2.05(2) 2.86(2) 169.0(18) 1_655 O7 H7 O1 0.82(2) 2.06(3) 2.86(3) 167(2) . O8 H8 O9 0.82(3) 2.14(4) 2.89(4) 153(3) 2_456 O9 H9 O8 0.82(3) 2.10(3) 2.89(4) 162(2) 2_446 O10 H10 O5 0.82(3) 2.02(3) 2.80(3) 159(3) . """)
def exercise(space_group_info, fixed_random_seed=True, shifted_origin=None, elements=None, d_min=0.8, grid_resolution_factor=1/3., verbose=False, **kwds): if elements is None: n_C = 5 n_O = 1 n_N = 1 elements = ["C"]*n_C + ["O"]*n_O + ["N"]*n_N if verbose: print elements target_space_group_type = space_group_info.type() hall = sgtbx.space_group_symbols( target_space_group_type.lookup_symbol()).hall() print hall if fixed_random_seed: random.seed(1) flex.set_random_seed(1) # Generate a random structure in real space, # compute its structure factors, # that we will try to recover the symmetry of target_structure = random_structure.xray_structure( space_group_info=space_group_info, elements=elements, use_u_iso=True, random_u_iso=True, random_u_iso_scale=0.04, use_u_aniso=False, ) if shifted_origin is None: shifted_origin = flex.random_double(3) shifted_origin = mat.col(shifted_origin) if verbose: print "new origin = (%.3f, %.3f, %.3f)" % shifted_origin.elems print target_structure_in_p1 = target_structure\ .expand_to_p1().apply_shift(shifted_origin) target_f_in_p1 = miller.build_set( crystal_symmetry=target_structure_in_p1, anomalous_flag=False, d_min=d_min ).structure_factors_from_scatterers( xray_structure=target_structure_in_p1, algorithm="direct").f_calc() # Recover space group? sf_symm = symmetry_search.structure_factor_symmetry(target_f_in_p1) if verbose: print sf_symm solution_hall, target_hall = [ sgi.as_reference_setting().type().hall_symbol() for sgi in (sf_symm.space_group_info, target_structure.space_group_info()) ] assert solution_hall == target_hall, (solution_hall, target_hall) # Shift maximises goodness of symmetry? gos, solution_f = sf_symm.symmetrised_structure_factors() if space_group_info.type().hall_symbol() != ' P 1': assert gos.correlation > 0.99 assert gos.gradient == (0, 0, 0) gos_away_from_max = sf_symm.symmetrised_structure_factors( delta=mat.col((0.1, 0.1, 0.1)))[0] assert gos_away_from_max.correlation < 0.9, gos_away_from_max.correlation # Recovered origin """The sequence of transform read: ----->target structure ^ V ^ V (1, shifted_origin=sigma) ^ V ^ shifted target ^ V ^ V sf_symm.cb_op_to_primitive = (P, 0) ^ V ^ shifted target in primitive cell = shifted solution in primitive cell ^ V ^ V (1, -sf_symm.origin=-s) ^ V ^ solution in primitive cell ^ V ^ V solution_to_target_cb_op = (Q, q) ^ V ^------------ The total transfrom from the target structure back to it reads (QP, q') with q' = (Q,q)(-s + P sigma) = (Q,q)delta_o with delta_o = sf_symm.cb_op_to_primitive(shifted_origin) - sf_symm.origin (Q, q') must leave the target structure space group invariant. Most of the time Q is the unit matrix and the test boils down to check whether delta is an allowed origin shift after changing to the input cell but it does not hurt to do the more general test all the time. """ solution_to_target_cb_op = ( target_structure.space_group_info() .change_of_basis_op_to_reference_setting().inverse() * sf_symm.space_group_info .change_of_basis_op_to_reference_setting()) if verbose: print print "solution -> target: %s" % solution_to_target_cb_op.as_xyz() delta_o = (mat.col(sf_symm.cb_op_to_primitive(shifted_origin)) - sf_symm.origin) delta = mat.col(solution_to_target_cb_op(delta_o)) stabilising_cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx( (solution_to_target_cb_op*sf_symm.cb_op_to_primitive).c().r(), sgtbx.tr_vec((delta*72).as_int()*2, tr_den=144))) # guarding against rounding errors on some platforms (e.g. FC8) # meaning that (delta*144).as_int() would not work. target_sg = target_structure.space_group() assert target_sg == target_sg.change_basis(stabilising_cb_op)
def try_den_weight_torsion(self, grid_pair): #backup_k_rep = self.params.tardy.\ # prolsq_repulsion_function_changes.k_rep local_seed = int(self.random_seed+grid_pair[1]) flex.set_random_seed(value=local_seed) random.seed(local_seed) self.fmodels.fmodel_xray().xray_structure.replace_scatterers( self.save_scatterers_local.deep_copy()) self.fmodels.update_xray_structure( xray_structure = self.fmodels.fmodel_xray().xray_structure, update_f_calc = True) utils.assert_xray_structures_equal( x1 = self.fmodels.fmodel_xray().xray_structure, x2 = self.model.xray_structure) gamma_local = grid_pair[0] weight_local = grid_pair[1] self.model.restraints_manager.geometry.\ den_manager.gamma = gamma_local self.model.restraints_manager.geometry.\ den_manager.weight = weight_local cycle = 0 self.model.restraints_manager.geometry.\ den_manager.current_cycle = cycle+1 num_den_cycles = self.model.restraints_manager.geometry.\ den_manager.num_cycles if self.params.den.optimize and \ self.nproc != Auto and \ self.nproc > 1: local_log = sys.stdout elif self.params.den.optimize and \ self.nproc == 1: if self.verbose: local_log = self.log else: local_log = StringIO() else: local_log = self.log print >> self.log, " ...trying gamma %.1f, weight %.1f" % ( gamma_local, weight_local) while cycle < num_den_cycles: #if self.model.restraints_manager.geometry.\ # generic_restraints_manager.den_manager.current_cycle == \ # self.model.restraints_manager.geometry.\ # generic_restraints_manager.den_manager.torsion_mid_point+1: # self.params.tardy.\ # prolsq_repulsion_function_changes.k_rep = 1.0 print >> local_log, "DEN cycle %d" % (cycle+1) #print >> local_log, "Random seed: %d" % flex.get_random_seed() r_free = self.fmodels.fmodel_xray().r_free() print >> local_log, "rfree at start of SA cycle: %.4f" % r_free print >> local_log, "k_rep = %.2f" % \ self.params.tardy.\ prolsq_repulsion_function_changes.k_rep tardy.run( fmodels=self.fmodels, model=self.model, target_weights=self.target_weights, params=self.params.tardy, log=local_log, format_for_phenix_refine=True, call_back_after_step=False) if self.params.den.bulk_solvent_and_scale: self.bulk_solvent_and_scale(log=local_log) self.fmodels.fmodel_xray().xray_structure = self.model.xray_structure if self.params.den.refine_adp: self.adp_refinement(log=local_log) self.model.restraints_manager.geometry.update_dihedral_ncs_restraints( sites_cart=self.model.xray_structure.sites_cart(), pdb_hierarchy=self.model.pdb_hierarchy(sync_with_xray_structure=True), log=local_log) cycle += 1 self.model.restraints_manager.geometry.\ den_manager.current_cycle += 1 r_free = self.fmodels.fmodel_xray().r_free() print >> local_log, "rfree at end of SA cycle: %f" % r_free r_free = self.fmodels.fmodel_xray().r_free() step_xray_structure = self.fmodels.fmodel_xray().\ xray_structure.deep_copy_scatterers().scatterers() step_eq_distances = self.model.restraints_manager.geometry.\ den_manager.get_current_eq_distances() return (gamma_local, weight_local, r_free, step_xray_structure, step_eq_distances)
from __future__ import division from cctbx.array_family import flex import mmtbx.f_model from cctbx.development import random_structure from cctbx.development import debug_utils from cctbx import sgtbx import random import sys from mmtbx.scaling import outlier_rejection from cctbx.xray import observation_types from cctbx.development import debug_utils if 1: random.seed(0) flex.set_random_seed(value=0) def exercise( d_min=3.5, k_sol=0.3, b_sol=60.0, b_cart=[0, 0, 0, 0, 0, 0], anomalous_flag=False, scattering_table="it1992", space_group_info=None, ): space_groups = [str(space_group_info)] for sg in space_groups: ### get random structure xray_structure = random_structure.xray_structure( space_group_info=sgtbx.space_group_info(sg),
def __init__(self, model, params, log = None): self.log = log self.params = params self.model = model self._neutralize_scatterers() if not model.crystal_symmetry() or not model.crystal_symmetry().unit_cell(): # Make it up from cctbx.maptbx.box import shift_and_box_model model = shift_and_box_model(model, shift_model=False) self.pdb_hierarchy = model.get_hierarchy() self.crystal_symmetry = model.crystal_symmetry() if(self.log is None): self.log = sys.stdout self.xray_structure = model.get_xray_structure() asc = self.pdb_hierarchy.atom_selection_cache( special_position_settings=crystal.special_position_settings( crystal_symmetry = self.crystal_symmetry)) if(self.params.random_seed is not None): random.seed(self.params.random_seed) flex.set_random_seed(self.params.random_seed) self.top_selection = flex.smart_selection( flags=flex.bool(self.xray_structure.scatterers().size(), True)) if(self.params.selection is not None): self.top_selection = flex.smart_selection( flags=asc.selection(self.params.selection)) self._rotate_about_axis() self._process_adp() self._process_sites() self._process_occupancies() self._put_in_box() self._change_of_basis() # Up to this point we are done with self.xray_structure self.model.set_xray_structure(self.xray_structure) self.pdb_hierarchy = self.model.get_hierarchy() # Now only manipulations that use self.pdb_hierarchy are done ### segID manipulations if (params.set_seg_id_to_chain_id): if (params.clear_seg_id): raise Sorry("Parameter conflict - set_seg_id_to_chain_id=True and "+ "clear_seg_id=True. Please choose only one of these options.") for atom in self.pdb_hierarchy.atoms(): labels = atom.fetch_labels() atom.segid = "%-4s" % labels.chain_id elif (params.clear_seg_id): for atom in self.pdb_hierarchy.atoms(): atom.segid = " " if(self.params.set_chemical_element_simple_if_necessary or self.params.rename_chain_id.old_id or self.params.renumber_residues or self.params.increment_resseq or self.params.convert_semet_to_met or self.params.convert_met_to_semet or self.params.set_charge.charge or self.params.truncate_to_polyala or self.params.truncate_to_polygly or self.params.remove_alt_confs or self.params.move_waters_last or self.params.remove_fraction or self.params.keep or self.params.remove): # del self.xray_structure # it is invalide below this point self._set_chemical_element_simple_if_necessary() self._rename_chain_id() self._renumber_residues() self._convert_semet_to_met() self._convert_met_to_semet() self._set_atomic_charge() self._truncate_to_poly_ala() self._truncate_to_poly_gly() self._remove_alt_confs() self._move_waters() self._remove_atoms() self._apply_keep_remove() # Here goes really nasty hack. Never repeat it. # It is here because I don't have clear idea about how to handle # such dramatic changes in number of atoms etc that just was performed # for hierarchy. self.pdb_hierarchy.reset_atom_i_seqs() self.pdb_hierarchy.atoms_reset_serial() self.model._pdb_hierarchy = self.pdb_hierarchy self.model._xray_structure = self.pdb_hierarchy.extract_xray_structure( crystal_symmetry=self.model.crystal_symmetry()) self.model._update_atom_selection_cache() self.model._update_has_hd() self.model.get_hierarchy().atoms().reset_i_seq()
def exercise(d_min=5, random_seed=1111111): inp = get_pdb_inputs(pdb_str=pdb_str) xrs_good = inp.xrs.deep_copy_scatterers() target_map = get_tmo(inp=inp, d_min=d_min) inp.ph.write_pdb_file(file_name="start.pdb") show(prefix="GOOD", pdb_hierarchy=inp.ph, tm=target_map, xrs=xrs_good, grm=inp.grm.geometry) # sites_cart_reference = [] selections_reference = [] pdb_hierarchy_reference = inp.ph.deep_copy() pdb_hierarchy_reference.reset_i_seq_if_necessary() for model in inp.ph.models(): for chain in model.chains(): for residue in chain.residues(): sites_cart_reference.append(residue.atoms().extract_xyz()) selections_reference.append(residue.atoms().extract_i_seq()) # sites_cart_reference_for_chi_only = [] selections_reference_for_chi_only = [] for model in inp.ph.models(): for chain in model.chains(): for residue in chain.residues(): s1 = flex.vec3_double() s2 = flex.size_t() for atom in residue.atoms(): if (not atom.name.strip().upper() in ["O"]): s1.append(atom.xyz) s2.append(atom.i_seq) sites_cart_reference_for_chi_only.append(s1) selections_reference_for_chi_only.append(s2) # xrs_poor = shake_sites(xrs=xrs_good.deep_copy_scatterers(), random=False, shift=2.0, grm=inp.grm) inp.ph.adopt_xray_structure(xrs_poor) inp.ph.write_pdb_file(file_name="poor.pdb") # for use_reference_torsion in [ "no", "yes_add_once", "yes_add_per_residue", "yes_manual" ]: es = inp.grm.energies_sites( sites_cart=xrs_good.sites_cart()) # it's essential to update grm inp.ph.adopt_xray_structure(xrs_poor) random.seed(random_seed) flex.set_random_seed(random_seed) print "*" * 79 print "use_reference_torsion:", use_reference_torsion print "*" * 79 show(prefix="START", pdb_hierarchy=inp.ph, tm=target_map, xrs=xrs_poor, grm=inp.grm.geometry) # if (use_reference_torsion == "yes_add_per_residue"): inp.grm.geometry.remove_chi_torsion_restraints_in_place() for sites_cart, selection in zip(sites_cart_reference, selections_reference): inp.grm.geometry.add_chi_torsion_restraints_in_place( pdb_hierarchy=pdb_hierarchy_reference, sites_cart=sites_cart, selection=selection, chi_angles_only=True, sigma=1) if (use_reference_torsion == "yes_add_once"): inp.grm.geometry.remove_chi_torsion_restraints_in_place() inp.grm.geometry.add_chi_torsion_restraints_in_place( pdb_hierarchy=pdb_hierarchy_reference, sites_cart=xrs_good.sites_cart(), chi_angles_only=True, sigma=1) if (use_reference_torsion == "yes_manual"): inp.grm.geometry.remove_chi_torsion_restraints_in_place() for sites_cart, selection in zip( sites_cart_reference_for_chi_only, selections_reference_for_chi_only): inp.grm.geometry.add_chi_torsion_restraints_in_place( pdb_hierarchy=pdb_hierarchy_reference, sites_cart=sites_cart, selection=selection, chi_angles_only=True, sigma=1) # tmp = xrs_poor.deep_copy_scatterers() rsr_simple_refiner = individual_sites.simple( target_map=target_map.data, selection=flex.bool(tmp.scatterers().size(), True), real_space_gradients_delta=d_min / 4, max_iterations=500, geometry_restraints_manager=inp.grm.geometry) refined = individual_sites.refinery(refiner=rsr_simple_refiner, optimize_weight=True, xray_structure=tmp, start_trial_weight_value=50, rms_bonds_limit=0.02, rms_angles_limit=2.0) assert refined.sites_cart_result is not None tmp = tmp.replace_sites_cart(refined.sites_cart_result) inp.ph.adopt_xray_structure(tmp) show(prefix="FINAL", pdb_hierarchy=inp.ph, tm=target_map, xrs=tmp, grm=inp.grm.geometry) inp.ph.write_pdb_file(file_name="final_%s.pdb" % str(use_reference_torsion))
from cctbx import xray import mmtbx.utils from scitbx.array_family import flex import random # For lbfgs class from cctbx import xray from cctbx import crystal from cctbx.array_family import flex import scitbx.lbfgs from libtbx import adopt_init_args from stdlib import math if (1): # fixed random seed random.seed(1) flex.set_random_seed(1) class lbfgs(object): def __init__(self, xray_structure, geometry_restraints, states, max_iterations=100, min_iterations=0, verbose=0, correct_special_position_tolerance=1.0): adopt_init_args(self, locals()) self.f = None self.correct_special_position_tolerance = correct_special_position_tolerance self.x = flex.double(self.xray_structure.n_parameters(), 0)
def run_test(params, pdb_files, other_files, callback=None, log=None): if (log is None): log = sys.stdout if (params.random_seed is not None): random.seed(params.random_seed) flex.set_random_seed(value=params.random_seed) # if (len(pdb_files) != 0): print >> log, "PDB files:" for file_name in pdb_files: print >> log, " ", file_name print >> log if (len(other_files) != 0): print >> log, "Other files:" for file_name in other_files: print >> log, " ", file_name print >> log # assert len(pdb_files) in [1, 2] # pdb_interpretation_params = pdb_interpretation.master_params.extract() pdb_interpretation_params.dihedral_function_type \ = params.dihedral_function_type processed_pdb_files = pdb_interpretation.run( args=pdb_files[-1:]+other_files, params=pdb_interpretation_params, strict_conflict_handling=False, substitute_non_crystallographic_unit_cell_if_necessary=True, return_all_processed_pdb_files=True, log=log) assert len(processed_pdb_files) == 1 print >> log # xs = processed_pdb_files[0].xray_structure() geo_manager = processed_pdb_files[0].geometry_restraints_manager() labels = [sc.label for sc in xs.scatterers()] ideal_sites_cart = xs.sites_cart() sites = ideal_sites_cart masses = xs.atomic_weights() tardy_tree_simple_connectivity = geo_manager.construct_tardy_tree(sites=sites) rmsd_calculator = tardy_tree_simple_connectivity.rmsd_calculator() # if (params.tardy_displacements is not None): def get_tardy_model_no_potential(): return scitbx.rigid_body.tardy_model( labels=labels, sites=sites, masses=masses, tardy_tree=tardy_tree_simple_connectivity, potential_obj=None) def get_tardy_model_no_density(): tardy_tree = scitbx.graph.tardy_tree.construct( n_vertices=len(sites), edge_list=[]) tardy_tree.build_tree() potential_obj = potential_object( density_map=None, geo_manager=geo_manager, reduced_geo_manager=None, prolsq_repulsion_function_changes= params.prolsq_repulsion_function_changes, real_space_gradients_delta=None, real_space_target_weight=None, ideal_sites_cart=None) return scitbx.rigid_body.tardy_model( labels=labels, sites=sites, masses=masses, tardy_tree=tardy_tree, potential_obj=potential_obj) if (params.tardy_displacements is Auto): auto_params = params.tardy_displacements_auto target_rmsd = \ params.structure_factors_high_resolution \ * auto_params.rmsd_vs_high_resolution_factor target_rmsd_tol = \ params.structure_factors_high_resolution \ * auto_params.rmsd_tolerance assert target_rmsd > 0 assert target_rmsd_tol > 0 print >> log, "Random displacements (%s):" \ % params.tardy_displacements_auto.parameterization print >> log, " high resolution: %.6g" \ % params.structure_factors_high_resolution print >> log, " target rmsd: %.6g" % target_rmsd print >> log, " target rmsd tolerance: %.6g" % target_rmsd_tol log.flush() def raise_max_steps_exceeded(var_name, rmsd_history): msg = [ "tardy_displacements_auto.max_steps exceeded:", " % -13s rmsd" % var_name] for var_rmsd in rmsd_history: msg.append(" %13.6e %13.6e" % var_rmsd) raise Sorry("\n".join(msg)) if (params.tardy_displacements_auto.parameterization == "cartesian"): multiplier = 1.5 rmsd_history = [] for i_step in xrange(auto_params.max_steps): sites = cartesian_random_displacements( sites_cart=ideal_sites_cart, target_rmsd=target_rmsd*multiplier) tardy_model = get_tardy_model_no_density() tardy_model.minimization(max_iterations=20) sites = tardy_model.sites_moved() sites_moved = sites rmsd = rmsd_calculator(sites_moved, ideal_sites_cart) rmsd_history.append((multiplier, rmsd)) print >> log, " multiplier, rmsd: %13.6e, %13.6e" \ % rmsd_history[-1] log.flush() if (rmsd < target_rmsd - target_rmsd_tol): if (rmsd != 0): multiplier = min( multiplier*2, max( multiplier*1.2, target_rmsd / rmsd)) else: multiplier *= 2 else: if (rmsd <= target_rmsd + target_rmsd_tol): tardy_model.minimization(max_iterations=500) sites = tardy_model.sites_moved() sites_moved = sites rmsd = rmsd_calculator(sites_moved, ideal_sites_cart) rmsd_history.append((0, rmsd)) print >> log, " multiplier, rmsd: %13.6e, %13.6e" \ % rmsd_history[-1] log.flush() break multiplier *= max(0.5, target_rmsd/rmsd) else: raise_max_steps_exceeded( var_name="multiplier", rmsd_history=rmsd_history) del rmsd_history print >> log, " actual rmsd: %.6g" % rmsd print >> log elif (params.tardy_displacements_auto.parameterization == "constrained"): tardy_model = get_tardy_model_no_potential() tardy_model.assign_random_velocities() delta_t = auto_params.first_delta_t rmsd_history = [] assert auto_params.max_steps > 0 for i_step in xrange(auto_params.max_steps): prev_q = tardy_model.pack_q() prev_qd = tardy_model.pack_qd() tardy_model.dynamics_step(delta_t=delta_t) sites_moved = tardy_model.sites_moved() rmsd = rmsd_calculator(sites_moved, ideal_sites_cart) rmsd_history.append((delta_t, rmsd)) if (rmsd < target_rmsd - target_rmsd_tol): delta_t *= 2 - rmsd / target_rmsd else: if (rmsd <= target_rmsd + target_rmsd_tol): break tardy_model.unpack_q(q_packed=prev_q) tardy_model.unpack_qd(qd_packed=prev_qd) delta_t *= 0.5 prev_q = None prev_qd = None else: raise_max_steps_exceeded( var_name="delta_t", rmsd_history=rmsd_history) del rmsd_history print >> log, " actual rmsd: %.6g" % rmsd print >> log, " tardy_displacements=%s" % ",".join( ["%.6g" % v for v in tardy_model.pack_q()]) print >> log sites = tardy_model.sites_moved() else: raise AssertionError else: tardy_model = get_tardy_model_no_potential() q = tardy_model.pack_q() if (len(params.tardy_displacements) != len(q)): print >> log, "tardy_displacements:", params.tardy_displacements hinge_edges = tardy_model.tardy_tree.cluster_manager.hinge_edges assert len(hinge_edges) == tardy_model.bodies_size() dofej = tardy_model.degrees_of_freedom_each_joint() qsej = tardy_model.q_size_each_joint() for ib,(i,j) in enumerate(hinge_edges): if (i == -1): si = "root" else: si = tardy_model.labels[i] sj = tardy_model.labels[j] print >> log, "%21s - %-21s: %d dof, %d q_size" % ( si, sj, dofej[ib], qsej[ib]) print >> log, "Zero displacements:" print >> log, " tardy_displacements=%s" % ",".join( [str(v) for v in q]) raise Sorry("Incompatible tardy_displacements.") tardy_model.unpack_q(q_packed=flex.double(params.tardy_displacements)) sites = tardy_model.sites_moved() # if (params.emulate_cartesian): tardy_tree = scitbx.graph.tardy_tree.construct( n_vertices=len(sites), edge_list=[]) tardy_tree.build_tree() else: tardy_tree = tardy_tree_simple_connectivity print >> log, "tardy_tree summary:" tardy_tree.show_summary(vertex_labels=labels, out=log, prefix=" ") print >> log # if (len(pdb_files) == 2): ideal_pdb_inp = iotbx.pdb.input(file_name=pdb_files[0]) ideal_pdb_hierarchy = ideal_pdb_inp.construct_hierarchy() assert ideal_pdb_hierarchy.is_similar_hierarchy( processed_pdb_files[0].all_chain_proxies.pdb_hierarchy) ideal_sites_cart = ideal_pdb_hierarchy.atoms().extract_xyz() xs.set_sites_cart(sites_cart=ideal_sites_cart) fft_map = xs.structure_factors( d_min=params.structure_factors_high_resolution).f_calc().fft_map() fft_map.apply_sigma_scaling() # assert not params.orca_experiments or not params.emulate_cartesian if (params.orca_experiments): from mmtbx.refinement import orca x = orca.expand( labels=labels, sites_cart=sites, masses=masses, geo_manager=geo_manager) labels = x.labels sites = x.sites_cart masses = x.masses geo_manager = x.geo_manager tardy_tree = x.tardy_tree x_ideal_sites_cart = flex.vec3_double() for i_orc,i_seq in x.indices: x_ideal_sites_cart.append(ideal_sites_cart[i_seq]) ideal_sites_cart = x_ideal_sites_cart rmsd_calculator = x.rmsd_calculator( tardy_tree_rmsd_calculator=rmsd_calculator) # if (params.emulate_cartesian or params.keep_all_restraints): reduced_geo_manager = None else: reduced_geo_manager = geo_manager.reduce_for_tardy(tardy_tree=tardy_tree) real_space_gradients_delta = \ params.structure_factors_high_resolution \ * params.real_space_gradients_delta_resolution_factor potential_obj = potential_object( density_map=fft_map.real_map(), geo_manager=geo_manager, reduced_geo_manager=reduced_geo_manager, prolsq_repulsion_function_changes=params.prolsq_repulsion_function_changes, real_space_gradients_delta=real_space_gradients_delta, real_space_target_weight=params.real_space_target_weight, ideal_sites_cart=ideal_sites_cart, site_labels=labels, orca_experiments=params.orca_experiments) tardy_model = scitbx.rigid_body.tardy_model( labels=labels, sites=sites, masses=masses, tardy_tree=tardy_tree, potential_obj=potential_obj) mmtbx.refinement.tardy.action( tardy_model=tardy_model, params=params, rmsd_calculator=rmsd_calculator, callback=callback, log=log) print >> log
import cctbx.geometry_restraints.flags import cctbx.geometry_restraints.manager import cctbx.geometry_restraints.lbfgs from cctbx import xray from cctbx import crystal import cctbx.crystal.coordination_sequences from cctbx import sgtbx from cctbx.array_family import flex from scitbx import matrix as mx import scitbx.lbfgs from libtbx.str_utils import format_value from itertools import count import sys if (1): flex.set_random_seed(0) class restraint_parameters(object): def __init__(self, distance_ideal, weight): self.distance_ideal = distance_ideal self.weight = weight restraint_parameters_si_o = restraint_parameters(1.61, 2.0) restraint_parameters_o_si_o = restraint_parameters(2.629099, 0.41) restraint_parameters_si_o_si = restraint_parameters(3.070969, 0.2308) def setup_bond_params_table(structure, bond_sym_table): scatterers = structure.scatterers() t = geometry_restraints.bond_params_table(scatterers.size()) for i_seq,bond_sym_dict in enumerate(bond_sym_table):
def exercise(d_min=5, random_seed=1111111): inp = get_pdb_inputs(pdb_str=pdb_str) xrs_good = inp.xrs.deep_copy_scatterers() target_map = get_tmo(inp=inp, d_min = d_min) inp.ph.write_pdb_file(file_name="start.pdb") show(prefix="GOOD", pdb_hierarchy = inp.ph, tm=target_map, xrs=xrs_good, grm=inp.grm.geometry) # sites_cart_reference = [] selections_reference = [] pdb_hierarchy_reference = inp.ph.deep_copy() pdb_hierarchy_reference.reset_i_seq_if_necessary() for model in inp.ph.models(): for chain in model.chains(): for residue in chain.residues(): sites_cart_reference.append(residue.atoms().extract_xyz()) selections_reference.append(residue.atoms().extract_i_seq()) # sites_cart_reference_for_chi_only = [] selections_reference_for_chi_only = [] for model in inp.ph.models(): for chain in model.chains(): for residue in chain.residues(): s1 = flex.vec3_double() s2 = flex.size_t() for atom in residue.atoms(): if(not atom.name.strip().upper() in ["O"]): s1.append(atom.xyz) s2.append(atom.i_seq) sites_cart_reference_for_chi_only.append(s1) selections_reference_for_chi_only.append(s2) # xrs_poor = shake_sites(xrs=xrs_good.deep_copy_scatterers(), random=False, shift=2.0, grm=inp.grm) inp.ph.adopt_xray_structure(xrs_poor) inp.ph.write_pdb_file(file_name="poor.pdb") # for use_reference_torsion in ["no", "yes_add_once", "yes_add_per_residue", "yes_manual"]: es = inp.grm.energies_sites(sites_cart = xrs_good.sites_cart()) # it's essential to update grm inp.ph.adopt_xray_structure(xrs_poor) random.seed(random_seed) flex.set_random_seed(random_seed) print "*"*79 print "use_reference_torsion:", use_reference_torsion print "*"*79 show(prefix="START",pdb_hierarchy = inp.ph, tm=target_map, xrs=xrs_poor, grm=inp.grm.geometry) # if(use_reference_torsion == "yes_add_per_residue"): inp.grm.geometry.remove_chi_torsion_restraints_in_place() for sites_cart, selection in zip(sites_cart_reference, selections_reference): inp.grm.geometry.add_chi_torsion_restraints_in_place( pdb_hierarchy = pdb_hierarchy_reference, sites_cart = sites_cart, selection = selection, chi_angles_only = True, sigma = 1) if(use_reference_torsion == "yes_add_once"): inp.grm.geometry.remove_chi_torsion_restraints_in_place() inp.grm.geometry.add_chi_torsion_restraints_in_place( pdb_hierarchy = pdb_hierarchy_reference, sites_cart = xrs_good.sites_cart(), chi_angles_only = True, sigma = 1) if(use_reference_torsion == "yes_manual"): inp.grm.geometry.remove_chi_torsion_restraints_in_place() for sites_cart, selection in zip(sites_cart_reference_for_chi_only, selections_reference_for_chi_only): inp.grm.geometry.add_chi_torsion_restraints_in_place( pdb_hierarchy = pdb_hierarchy_reference, sites_cart = sites_cart, selection = selection, chi_angles_only = True, sigma = 1) # tmp = xrs_poor.deep_copy_scatterers() rsr_simple_refiner = individual_sites.simple( target_map = target_map.data, selection = flex.bool(tmp.scatterers().size(), True), real_space_gradients_delta = d_min/4, max_iterations = 500, geometry_restraints_manager = inp.grm.geometry) refined = individual_sites.refinery( refiner = rsr_simple_refiner, optimize_weight = True, xray_structure = tmp, start_trial_weight_value = 50, rms_bonds_limit = 0.02, rms_angles_limit = 2.0) assert refined.sites_cart_result is not None tmp = tmp.replace_sites_cart(refined.sites_cart_result) inp.ph.adopt_xray_structure(tmp) show(prefix="FINAL",pdb_hierarchy = inp.ph, tm=target_map, xrs=tmp, grm=inp.grm.geometry) inp.ph.write_pdb_file(file_name="final_%s.pdb"%str(use_reference_torsion))
from __future__ import division import clipper from cctbx import xray from cctbx import miller from cctbx import crystal from cctbx.development import random_structure from cctbx.development import debug_utils from cctbx.array_family import flex from libtbx.test_utils import approx_equal import sys if (1): flex.set_random_seed(2) def exercise_SFweight_spline_core(structure, d_min, verbose=0): structure.scattering_type_registry(d_min=d_min) f_obs = abs(structure.structure_factors( d_min=d_min, anomalous_flag=False).f_calc()) if (0 or verbose): f_obs.show_summary() f_obs = miller.array( miller_set=f_obs, data=f_obs.data(), sigmas=flex.sqrt(f_obs.data())) partial_structure = xray.structure( crystal_symmetry=structure, scatterers=structure.scatterers()[:-2]) f_calc = f_obs.structure_factors_from_scatterers( xray_structure=partial_structure).f_calc() test_set_flags = (flex.random_double(size=f_obs.indices().size()) < 0.1) sfweight = clipper.SFweight_spline_interface(
def exercise_restrained_refinement(options): import random random.seed(1) flex.set_random_seed(1) xs0 = smtbx.development.random_xray_structure( sgtbx.space_group_info('P1'), n_scatterers=options.n_scatterers, elements="random") for sc in xs0.scatterers(): sc.flags.set_grad_site(True) sc0 = xs0.scatterers() uc = xs0.unit_cell() mi = xs0.build_miller_set(anomalous_flag=False, d_min=options.resolution) fo_sq = mi.structure_factors_from_scatterers( xs0, algorithm="direct").f_calc().norm() fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1)) i, j, k, l = random.sample(xrange(options.n_scatterers), 4) bond_proxies = geometry_restraints.shared_bond_simple_proxy() w = 1e9 d_ij = uc.distance(sc0[i].site, sc0[j].site) * 0.8 bond_proxies.append( geom.bond_simple_proxy(i_seqs=(i, j), distance_ideal=d_ij, weight=w)) d_jk = uc.distance(sc0[j].site, sc0[k].site) * 0.85 bond_proxies.append( geom.bond_simple_proxy(i_seqs=(j, k), distance_ideal=d_jk, weight=w)) d_ki = min( uc.distance(sc0[k].site, sc0[i].site) * 0.9, (d_ij + d_jk) * 0.8) bond_proxies.append( geom.bond_simple_proxy(i_seqs=(k, i), distance_ideal=d_ki, weight=w)) d_jl = uc.distance(sc0[j].site, sc0[l].site) * 0.9 bond_proxies.append( geom.bond_simple_proxy(i_seqs=(j, l), distance_ideal=d_jl, weight=w)) d_lk = min( uc.distance(sc0[l].site, sc0[k].site) * 0.8, 0.75 * (d_jk + d_jl)) bond_proxies.append( geom.bond_simple_proxy(i_seqs=(l, k), distance_ideal=d_jl, weight=w)) restraints_manager = restraints.manager(bond_proxies=bond_proxies) xs1 = xs0.deep_copy_scatterers() xs1.shake_sites_in_place(rms_difference=0.1) def ls_problem(): xs = xs1.deep_copy_scatterers() reparametrisation = constraints.reparametrisation( structure=xs, constraints=[], connectivity_table=smtbx.utils.connectivity_table(xs), temperature=20) return least_squares.crystallographic_ls( fo_sq.as_xray_observations(), reparametrisation=reparametrisation, restraints_manager=restraints_manager) gradient_threshold, step_threshold = 1e-6, 1e-6 eps = 5e-3 ls = ls_problem() t = wall_clock_time() cycles = normal_eqns_solving.naive_iterations( ls, gradient_threshold=gradient_threshold, step_threshold=step_threshold, track_all=True) if options.verbose: print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed()) sc = ls.xray_structure.scatterers() for p in bond_proxies: d = uc.distance(*[sc[i_pair].site for i_pair in p.i_seqs]) assert approx_equal(d, p.distance_ideal, eps) ls = ls_problem() t = wall_clock_time() cycles = normal_eqns_solving.levenberg_marquardt_iterations( ls, gradient_threshold=gradient_threshold, step_threshold=step_threshold, tau=1e-3, track_all=True) if options.verbose: print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed()) sc = ls.xray_structure.scatterers() sc = ls.xray_structure.scatterers() for p in bond_proxies: d = uc.distance(*[sc[i].site for i in p.i_seqs]) assert approx_equal(d, p.distance_ideal, eps)
def try_den_weight_cartesian(self, grid_pair): local_seed = int(self.random_seed+grid_pair[1]) flex.set_random_seed(value=local_seed) random.seed(local_seed) self.fmodels.fmodel_xray().xray_structure.replace_scatterers( self.save_scatterers_local.deep_copy()) self.fmodels.update_xray_structure( xray_structure = self.fmodels.fmodel_xray().xray_structure, update_f_calc = True) utils.assert_xray_structures_equal( x1 = self.fmodels.fmodel_xray().xray_structure, x2 = self.model.xray_structure) gamma_local = grid_pair[0] weight_local = grid_pair[1] self.model.restraints_manager.geometry.\ den_manager.gamma = gamma_local self.model.restraints_manager.geometry.\ den_manager.weight = weight_local cycle = 0 self.model.restraints_manager.geometry.\ den_manager.current_cycle = cycle+1 num_den_cycles = self.model.restraints_manager.geometry.\ den_manager.num_cycles if self.params.den.optimize and \ self.nproc != Auto and \ self.nproc > 1: local_log = sys.stdout elif self.params.den.optimize and \ self.nproc == 1: if self.verbose: local_log = self.log else: local_log = StringIO() else: local_log = self.log print >> self.log, " ...trying gamma %f, weight %f" % ( gamma_local, weight_local) while cycle < num_den_cycles: print >> local_log, "DEN cycle %s" % (cycle+1) r_free = self.fmodels.fmodel_xray().r_free() print >> local_log, "rfree at start of SA cycle: %f" % r_free simulated_annealing.manager( params = self.params.simulated_annealing, target_weights = self.target_weights, macro_cycle = self.macro_cycle, h_params = self.params.hydrogens, fmodels = self.fmodels, model = self.model, all_params = self.params, out = local_log) if self.params.den.bulk_solvent_and_scale: self.bulk_solvent_and_scale(log=local_log) if self.params.den.refine_adp: self.adp_refinement(log=local_log) self.model.restraints_manager.geometry.update_dihedral_ncs_restraints( sites_cart=self.model.xray_structure.sites_cart(), pdb_hierarchy=self.model.pdb_hierarchy(sync_with_xray_structure=True), log=local_log) cycle += 1 self.model.restraints_manager.geometry.\ den_manager.current_cycle += 1 r_free = self.fmodels.fmodel_xray().r_free() print >> local_log, "rfree at end of SA cycle: %f" % r_free r_free = self.fmodels.fmodel_xray().r_free() step_xray_structure = self.fmodels.fmodel_xray().\ xray_structure.deep_copy_scatterers().scatterers() step_eq_distances = self.model.restraints_manager.geometry.\ den_manager.get_current_eq_distances() return (gamma_local, weight_local, r_free, step_xray_structure, step_eq_distances)
def exercise_6_instantiate_consistency(symbol="C 2"): random.seed(0) flex.set_random_seed(0) for scale in [1.e-4, 1.0, 1.e+4]: for k_sol in [0, 0.3]: for b_sol in [0, 50]: for set_h_occ_to_zero in [True, False]: for update_f_part1 in [True, False]: for apply_scale_to in ["f_obs", "f_model"]: # Simulate Fobs START x = random_structure.xray_structure( space_group_info=sgtbx.space_group_info( symbol=symbol), elements=(("O", "N", "C") * 3 + ("H", ) * 10), volume_per_atom=50, min_distance=3, general_positions_only=True, random_u_iso=True, random_occupancy=False) x.scattering_type_registry(table="wk1995") x.set_occupancies(value=0.8, selection=x.hd_selection()) f_calc = x.structure_factors(d_min=2.0).f_calc() mask_manager = mmtbx.masks.manager( miller_array=f_calc) f_mask = mask_manager.shell_f_masks( xray_structure=x)[0] assert flex.mean(abs(f_mask).data()) > 0 b_cart = adptbx.random_traceless_symmetry_constrained_b_cart( crystal_symmetry=x.crystal_symmetry()) u_star = adptbx.u_cart_as_u_star( x.unit_cell(), adptbx.b_as_u(b_cart)) k_anisotropic = mmtbx.f_model.ext.k_anisotropic( f_calc.indices(), u_star) ss = 1. / flex.pow2( f_calc.d_spacings().data()) / 4. k_mask = mmtbx.f_model.ext.k_mask(ss, k_sol, b_sol) if (apply_scale_to == "f_model"): k_isotropic = flex.double( f_calc.data().size(), scale) else: k_isotropic = flex.double( f_calc.data().size(), 1) f_model_data = scale * k_anisotropic * ( f_calc.data() + k_mask * f_mask.data()) f_model = f_calc.customized_copy(data=f_model_data) f_obs = abs(f_model) if (apply_scale_to == "f_obs"): f_obs = f_obs.customized_copy( data=f_obs.data() * scale) r_free_flags = f_obs.generate_r_free_flags() # Simulate Fobs END if (set_h_occ_to_zero): x.set_occupancies(value=0.0, selection=x.hd_selection()) x.shake_sites_in_place(mean_distance=5) sel = x.random_remove_sites_selection(fraction=0.3) x = x.select(sel) fmodel = mmtbx.f_model.manager( xray_structure=x, f_obs=f_obs, r_free_flags=r_free_flags) fmodel.update_all_scales( fast=True, show=False, update_f_part1=update_f_part1) f_part1_data = fmodel.f_calc().data( ) * flex.random_double( fmodel.f_calc().data().size()) f_part1 = fmodel.f_calc().customized_copy( data=f_part1_data) fmodel.update(f_part1=f_part1) r1 = fmodel.r_work() # zero = fmodel.f_calc().customized_copy( data=fmodel.f_calc().data() * 0) fmodel_dc = mmtbx.f_model.manager( f_obs=fmodel.f_obs(), r_free_flags=fmodel.r_free_flags(), k_isotropic=fmodel.k_isotropic(), k_anisotropic=fmodel.k_anisotropic(), f_calc=fmodel.f_model_no_scales(), f_part1=fmodel.f_part1(), f_part2=fmodel.f_part2(), f_mask=zero) r2 = fmodel_dc.r_work() if (0): print("r1=%8.6f r2=%8.6f fp1=%6.3f fp2=%6.3f fc=%6.3f"%(r1, r2, flex.mean(abs(fmodel.f_part1()).data()), \ flex.mean(abs(fmodel.f_part2()).data()), \ flex.mean(abs(fmodel.f_calc()).data())), \ "set_h_occ_to_zero=", set_h_occ_to_zero,\ "update_f_part1=", update_f_part1) assert approx_equal(r1, r2), [r1, r2]
import math, sys, random from libtbx.test_utils import approx_equal from libtbx.utils import format_cpu_times import random, copy from cctbx import sgtbx from cctbx import adptbx from cctbx.development import random_structure import scitbx.math from mmtbx.tls import tools from mmtbx_tls_ext import * from six.moves import range if (1): # fixed random seed to avoid rare failures random.seed(0) flex.set_random_seed(0) def branch_3_mn(): m = None n = None small = 1.e-15 for i in range(10000): m2 = [] n2 = [] for i in range(4): r1 = random.random() r2 = random.random() r3 = random.random() if (r3 > 0.1): r1 = r2
def exercise_6_instantiate_consistency(symbol = "C 2"): random.seed(0) flex.set_random_seed(0) for scale in [1.e-4, 1.0, 1.e+4]: for k_sol in [0, 0.3]: for b_sol in [0, 50]: for set_h_occ_to_zero in [True, False]: for update_f_part1 in [True, False]: for apply_scale_to in ["f_obs", "f_model"]: # Simulate Fobs START x = random_structure.xray_structure( space_group_info = sgtbx.space_group_info(symbol=symbol), elements =(("O","N","C")*3+("H",)*10), volume_per_atom = 50, min_distance = 3, general_positions_only = True, random_u_iso = True, random_occupancy = False) x.scattering_type_registry(table="wk1995") x.set_occupancies(value=0.8, selection = x.hd_selection()) f_calc = x.structure_factors(d_min = 2.0).f_calc() mask_manager = mmtbx.masks.manager(miller_array = f_calc) f_mask = mask_manager.shell_f_masks(xray_structure = x)[0] assert flex.mean(abs(f_mask).data()) > 0 b_cart=adptbx.random_traceless_symmetry_constrained_b_cart( crystal_symmetry=x.crystal_symmetry()) u_star = adptbx.u_cart_as_u_star(x.unit_cell(), adptbx.b_as_u(b_cart)) k_anisotropic = mmtbx.f_model.ext.k_anisotropic(f_calc.indices(), u_star) ss = 1./flex.pow2(f_calc.d_spacings().data()) / 4. k_mask = mmtbx.f_model.ext.k_mask(ss, k_sol, b_sol) if(apply_scale_to=="f_model"): k_isotropic = flex.double(f_calc.data().size(), scale) else: k_isotropic = flex.double(f_calc.data().size(), 1) f_model_data = scale*k_anisotropic*(f_calc.data()+k_mask*f_mask.data()) f_model = f_calc.customized_copy(data = f_model_data) f_obs = abs(f_model) if(apply_scale_to=="f_obs"): f_obs = f_obs.customized_copy(data = f_obs.data()*scale) r_free_flags = f_obs.generate_r_free_flags() # Simulate Fobs END if(set_h_occ_to_zero): x.set_occupancies(value=0.0, selection = x.hd_selection()) x.shake_sites_in_place(mean_distance=5) sel = x.random_remove_sites_selection(fraction=0.3) x = x.select(sel) fmodel = mmtbx.f_model.manager( xray_structure = x, f_obs = f_obs, r_free_flags = r_free_flags) fmodel.update_all_scales(fast=True, show=False, update_f_part1=update_f_part1) f_part1_data = fmodel.f_calc().data()*flex.random_double( fmodel.f_calc().data().size()) f_part1 = fmodel.f_calc().customized_copy(data = f_part1_data) fmodel.update(f_part1 = f_part1) r1 = fmodel.r_work() # zero=fmodel.f_calc().customized_copy(data=fmodel.f_calc().data()*0) fmodel_dc = mmtbx.f_model.manager( f_obs = fmodel.f_obs(), r_free_flags = fmodel.r_free_flags(), k_isotropic = fmodel.k_isotropic(), k_anisotropic = fmodel.k_anisotropic(), f_calc = fmodel.f_model_no_scales(), f_part1 = fmodel.f_part1(), f_part2 = fmodel.f_part2(), f_mask = zero) r2 = fmodel_dc.r_work() if(0): print "r1=%8.6f r2=%8.6f fp1=%6.3f fp2=%6.3f fc=%6.3f"%(r1, r2, flex.mean(abs(fmodel.f_part1()).data()), \ flex.mean(abs(fmodel.f_part2()).data()), \ flex.mean(abs(fmodel.f_calc()).data())), \ "set_h_occ_to_zero=", set_h_occ_to_zero,\ "update_f_part1=", update_f_part1 assert approx_equal(r1, r2), [r1, r2]