def __init__(self, sweep, params):
self.params = params
flex.set_random_seed(params.random_seed)
unit_cell = params.unit_cell
assert unit_cell is not None
sgi = params.space_group
if sgi is None:
sgi = sgtbx.space_group_info(symbol="P 1")
B = scitbx.matrix.sqr(unit_cell.fractionalization_matrix()).transpose()
U = scitbx.matrix.sqr(flex.random_double_r3_rotation_matrix())
direct_matrix = (U * B).inverse()
crystal_model = Crystal(direct_matrix[0:3],
direct_matrix[3:6],
direct_matrix[6:9],
space_group=sgi.group())
scan = sweep.get_scan()
angle = self.params.rotation_angle
scan.set_image_range((1, iceil(angle/scan.get_oscillation()[1])))
predicted = predict_reflections(sweep, crystal_model)
beam_vectors = predicted.beam_vector()
S = beam_vectors - sweep.get_beam().get_s0()
centroids = S.rotate_around_origin(sweep.get_goniometer().get_rotation_axis(),
-predicted.rotation_angle())
self.d_min = self.params.reciprocal_space_grid.d_min
self.gridding = tuple([self.params.reciprocal_space_grid.n_points]*3)
centroids = centroids.select((1/centroids.norms())>=self.d_min)
assert len(centroids) > 0
self.map_to_grid(sweep, centroids)
self.fft()
debug_write_reciprocal_lattice_points_as_pdb(centroids)
self.debug_write_ccp4_map(self.grid_real, "fft.map")
def exercise_f_model_no_scales(symbol = "C 2"):
random.seed(0)
flex.set_random_seed(0)
x = random_structure.xray_structure(
space_group_info = sgtbx.space_group_info(symbol=symbol),
elements =(("O","N","C")*5+("H",)*10),
volume_per_atom = 200,
min_distance = 1.5,
general_positions_only = True,
random_u_iso = True,
random_occupancy = False)
f_obs = abs(x.structure_factors(d_min = 1.5, algorithm="fft").f_calc())
x.shake_sites_in_place(mean_distance=1)
k_iso = flex.double(f_obs.data().size(), 2)
k_aniso = flex.double(f_obs.data().size(), 3)
fmodel = mmtbx.f_model.manager(
xray_structure = x,
k_isotropic = k_iso,
k_anisotropic = k_aniso,
f_obs = f_obs)
fc = abs(fmodel.f_calc()).data()
fm = abs(fmodel.f_model()).data()
fmns = abs(fmodel.f_model_no_scales()).data()
assert approx_equal(flex.mean(fm/fc), 6)
assert approx_equal(flex.mean(fmns/fc), 1)
def __init__(self, sweep, params):
self.params = params
flex.set_random_seed(params.random_seed)
unit_cell = params.unit_cell
assert unit_cell is not None
sgi = params.space_group
if sgi is None:
sgi = sgtbx.space_group_info(symbol="P 1")
B = scitbx.matrix.sqr(unit_cell.fractionalization_matrix()).transpose()
U = scitbx.matrix.sqr(flex.random_double_r3_rotation_matrix())
direct_matrix = (U * B).inverse()
crystal_model = Crystal(
direct_matrix[0:3],
direct_matrix[3:6],
direct_matrix[6:9],
space_group=sgi.group(),
)
scan = sweep.get_scan()
angle = self.params.rotation_angle
scan.set_image_range((1, iceil(angle / scan.get_oscillation()[1])))
predicted = predict_reflections(sweep, crystal_model)
beam_vectors = predicted.beam_vector()
S = beam_vectors - sweep.get_beam().get_s0()
centroids = S.rotate_around_origin(
sweep.get_goniometer().get_rotation_axis(), -predicted.rotation_angle()
)
self.d_min = self.params.reciprocal_space_grid.d_min
self.gridding = tuple([self.params.reciprocal_space_grid.n_points] * 3)
centroids = centroids.select((1 / centroids.norms()) >= self.d_min)
assert len(centroids) > 0
self.map_to_grid(sweep, centroids)
self.fft()
debug_write_reciprocal_lattice_points_as_pdb(centroids)
self.debug_write_ccp4_map(self.grid_real, "fft.map")
def run():
libtbx.utils.show_times_at_exit()
import sys
from libtbx.option_parser import option_parser
command_line = (option_parser()
.option(None, "--fix_random_seeds",
action="store_true",
default=False)
.option(None, "--runs",
type='int',
default=1)
.option(None, "--verbose",
action="store_true",
default=False)
.option(None, "--skip-twin-test",
dest='skip_twin_test',
action="store_true",
default=False)
).process(args=sys.argv[1:])
if command_line.options.fix_random_seeds:
flex.set_random_seed(1)
random.seed(2)
n_runs = command_line.options.runs
if n_runs > 1: refinement_test.ls_cycle_repeats = n_runs
exercise_normal_equations()
exercise_floating_origin_dynamic_weighting(command_line.options.verbose)
special_positions_test(n_runs).run()
if not command_line.options.skip_twin_test:
twin_test().run()
def run(args):
random_seed = None
forever = False
for arg in args:
if (arg.startswith("--random_seed=")):
random_seed = int(arg.split("=", 1)[1])
if (arg.startswith("--forever")):
forever = True
if forever: assert random_seed is None
if random_seed is not None: assert forever == False
while 1:
if (random_seed is None):
random_seed = flex.get_random_seed()
print "random_seed:", random_seed
sys.stdout.flush()
random.seed(random_seed)
flex.set_random_seed(value=random_seed)
calculate_fobs()
exercise_1()
exercise_2()
check_result()
sys.stdout.flush()
if(not forever):
print "random_seed last used:", random_seed
break
def run():
libtbx.utils.show_times_at_exit()
import sys
from libtbx.option_parser import option_parser
command_line = (option_parser()
.option(None, "--fix_random_seeds",
action="store_true",
default=False)
.option(None, "--runs",
type='int',
default=1)
.option(None, "--verbose",
action="store_true",
default=False)
).process(args=sys.argv[1:])
if command_line.options.fix_random_seeds:
flex.set_random_seed(1)
random.seed(2)
n_runs = command_line.options.runs
if n_runs > 1: refinement_test.ls_cycle_repeats = n_runs
exercise_normal_equations()
exercise_floating_origin_dynamic_weighting(command_line.options.verbose)
special_positions_test(n_runs).run()
twin_test().run()
def get_sf(k_sol,
b_sol,
b_cart,
xrs,
miller_set=None,
d_min=None,
twin_law=None,
sfg_params=None):
random.seed(0)
flex.set_random_seed(0)
if (miller_set is None):
assert d_min is not None
f_dummy = abs(
xrs.structure_factors(d_min=d_min, anomalous_flag=False).f_calc())
else:
f_dummy = miller_set
assert d_min is None
r_free_flags = f_dummy.generate_r_free_flags(fraction=0.1)
fmodel = mmtbx.f_model.manager(r_free_flags=r_free_flags,
f_obs=f_dummy,
sf_and_grads_accuracy_params=sfg_params,
xray_structure=xrs,
twin_law=twin_law)
ss = 1. / flex.pow2(r_free_flags.d_spacings().data()) / 4.
k_mask = mmtbx.f_model.ext.k_mask(ss, k_sol, b_sol)
u_star = adptbx.u_cart_as_u_star(xrs.unit_cell(), adptbx.b_as_u(b_cart))
k_anisotropic = mmtbx.f_model.ext.k_anisotropic(r_free_flags.indices(),
u_star)
fmodel.update_xray_structure(xray_structure=xrs,
update_f_calc=True,
update_f_mask=True)
fmodel.update_core(k_mask=k_mask, k_anisotropic=k_anisotropic)
f_obs = abs(fmodel.f_model())
return f_obs, r_free_flags
def exercise_f_model_no_scales(symbol = "C 2"):
random.seed(0)
flex.set_random_seed(0)
x = random_structure.xray_structure(
space_group_info = sgtbx.space_group_info(symbol=symbol),
elements =(("O","N","C")*5+("H",)*10),
volume_per_atom = 200,
min_distance = 1.5,
general_positions_only = True,
random_u_iso = True,
random_occupancy = False)
f_obs = abs(x.structure_factors(d_min = 1.5, algorithm="fft").f_calc())
x.shake_sites_in_place(mean_distance=1)
k_iso = flex.double(f_obs.data().size(), 2)
k_aniso = flex.double(f_obs.data().size(), 3)
fmodel = mmtbx.f_model.manager(
xray_structure = x,
k_isotropic = k_iso,
k_anisotropic = k_aniso,
f_obs = f_obs)
fc = abs(fmodel.f_calc()).data()
fm = abs(fmodel.f_model()).data()
fmns = abs(fmodel.f_model_no_scales()).data()
assert approx_equal(flex.mean(fm/fc), 6)
assert approx_equal(flex.mean(fmns/fc), 1)
def run(args):
random_seed = None
forever = False
for arg in args:
if (arg.startswith("--random_seed=")):
random_seed = int(arg.split("=", 1)[1])
if (arg.startswith("--forever")):
forever = True
if forever: assert random_seed is None
if random_seed is not None: assert forever == False
while 1:
if (random_seed is None):
random_seed = flex.get_random_seed()
print "random_seed:", random_seed
sys.stdout.flush()
random.seed(random_seed)
flex.set_random_seed(value=random_seed)
calculate_fobs()
exercise_1()
exercise_2()
check_result()
sys.stdout.flush()
if (not forever):
print "random_seed last used:", random_seed
break
def exercise(flags, space_group_info):
if not flags.repeats: flags.repeats = 1
if not flags.algo: flags.algo = "weak_reflection_improved"
if not flags.on: flags.on = "E"
if flags.fix_seed:
random.seed(1)
flex.set_random_seed(1)
n = len(space_group_info.group())
print space_group_info.type().hall_symbol(),
if not flags.high_symmetry and n > 24:
print ' [ skipped ]'
if flags.Verbose: print
return
else:
print
n_C = 12 // n or 1
n_O = 6 // n
n_N = 3 // n
if flags.Verbose:
print "unit cell content: C%i O%i N%i" % (n_C * n, n_O * n, n_N * n)
print "asu content: C%i O%i N%i" % (n_C, n_O, n_N)
print "on %s's with %s" % (flags.on, flags.algo)
flipping_type = eval("charge_flipping.%s_iterator" % flags.algo)
for i in xrange(int(flags.repeats)):
randomly_exercise(flipping_type=flipping_type,
space_group_info=space_group_info,
elements=["C"] * n_C + ["O"] * n_O + ["N"] * n_N,
anomalous_flag=False,
d_min=0.8,
verbose=flags.Verbose,
amplitude_type=flags.on)
if flags.Verbose: print
def exercise():
import random
if (1): # fixed random seed to avoid rare failures
random.seed(0)
flex.set_random_seed(0)
verbose = "--verbose" in sys.argv[1:]
for n in [4, 10, 20, 100, 200]:
exercise_core(n=n, verbose=verbose)
print "OK"
def run(args):
if "--fix_random_seeds" in args:
random.seed(1)
flex.set_random_seed(1)
scitbx.random.set_random_seed(1)
libtbx.utils.show_times_at_exit()
for i in range(10):
exercise_optimise_shelxl_weights()
exercise_weighting_schemes()
def run(args):
if "--fix_random_seeds" in args:
random.seed(1)
flex.set_random_seed(1)
scitbx.random.set_random_seed(1)
libtbx.utils.show_times_at_exit()
for i in range(10):
exercise_optimise_shelxl_weights()
exercise_weighting_schemes()
def exercise():
import random
if 1: # fixed random seed to avoid rare failures
random.seed(0)
flex.set_random_seed(0)
verbose = "--verbose" in sys.argv[1:]
for n in [4, 10, 20, 100, 200]:
exercise_core(n=n, verbose=verbose)
print "OK"
def exercise_unmerged () :
quartz_structure = xray.structure(
special_position_settings=crystal.special_position_settings(
crystal_symmetry=crystal.symmetry(
unit_cell=(5.01,5.01,5.47,90,90,120),
space_group_symbol="P6222")),
scatterers=flex.xray_scatterer([
xray.scatterer(
label="Si",
site=(1/2.,1/2.,1/3.),
u=0.2),
xray.scatterer(
label="O",
site=(0.197,-0.197,0.83333),
u=0.1)]))
quartz_structure.set_inelastic_form_factors(
photon=1.54,
table="sasaki")
fc = abs(quartz_structure.structure_factors(d_min=1.0).f_calc())
symm = fc.crystal_symmetry()
icalc = fc.expand_to_p1().f_as_f_sq().set_observation_type_xray_intensity()
# generate 'unmerged' data
i_obs = icalc.customized_copy(crystal_symmetry=symm)
# now make up sigmas and some (hopefully realistic) error
flex.set_random_seed(12345)
n_refl = i_obs.size()
sigmas = flex.random_double(n_refl) * flex.mean(fc.data())
sigmas = icalc.customized_copy(data=sigmas).apply_debye_waller_factors(
u_iso=0.15)
err = (flex.double(n_refl, 0.5) - flex.random_double(n_refl)) * 2
i_obs = i_obs.customized_copy(
sigmas=sigmas.data(),
data=i_obs.data() + err)
# check for unmerged acentrics
assert i_obs.is_unmerged_intensity_array()
i_obs_centric = i_obs.select(i_obs.centric_flags().data())
i_obs_acentric = i_obs.select(~(i_obs.centric_flags().data()))
i_mrg_acentric = i_obs_acentric.merge_equivalents().array()
i_mixed = i_mrg_acentric.concatenate(i_obs_centric)
assert not i_mixed.is_unmerged_intensity_array()
# XXX These results of these functions are heavily dependent on the
# behavior of the random number generator, which is not consistent across
# platforms - therefore we can only check for very approximate values.
# Exact numerical results are tested with real data (stored elsewhere).
# CC1/2, etc.
assert approx_equal(i_obs.cc_one_half(), 0.9998, eps=0.001)
assert i_obs.resolution_filter(d_max=1.2).cc_one_half() > 0
assert i_obs.cc_anom() > 0.1
r_ano = i_obs.r_anom()
assert approx_equal(r_ano, 0.080756, eps=0.0001)
# merging stats
i_mrg = i_obs.merge_equivalents()
assert i_mrg.r_merge() < 0.1
assert i_mrg.r_meas() < 0.1
assert i_mrg.r_pim() < 0.05
def run_call_back(flags, space_group_info):
if not space_group_info.group().is_centric():
if flags.fix_random_seeds:
random.seed(1)
flex.set_random_seed(1)
scitbx.random.set_random_seed(1)
hooft_analysis_test_case(
space_group_info,use_students_t_errors=False).exercise(debug=flags.Debug)
if students_t_available:
hooft_analysis_test_case(
space_group_info,use_students_t_errors=True).exercise(debug=flags.Debug)
def run_call_back(flags, space_group_info):
if not space_group_info.group().is_centric():
if flags.fix_random_seeds:
random.seed(1)
flex.set_random_seed(1)
scitbx.random.set_random_seed(1)
hooft_analysis_test_case(
space_group_info,use_students_t_errors=False).exercise(debug=flags.Debug)
if students_t_available:
hooft_analysis_test_case(
space_group_info,use_students_t_errors=True).exercise(debug=flags.Debug)
def exercise(self, xray_structure=None, space_group_info=None, verbose=False, fixed_random_seed=True, **kwds):
assert [xray_structure, space_group_info].count(None) == 1
if xray_structure is None:
self.xs = self.random_structure(space_group_info, set_grads=True)
else:
self.xs = xray_structure
if fixed_random_seed:
random.seed(1)
flex.set_random_seed(1)
self.do_exercise(verbose)
def __init__(self, xray_structure, params, pdb_hierarchy, log=None):
self.log = log
self.pdb_hierarchy = pdb_hierarchy
self.crystal_symmetry = xray_structure.crystal_symmetry()
if (self.log is None): self.log = sys.stdout
self.xray_structure = xray_structure
self.params = params
asc = self.pdb_hierarchy.atom_selection_cache(
special_position_settings=crystal.special_position_settings(
crystal_symmetry=self.crystal_symmetry))
if (self.params.random_seed is not None):
random.seed(self.params.random_seed)
flex.set_random_seed(self.params.random_seed)
self.top_selection = flex.smart_selection(
flags=flex.bool(xray_structure.scatterers().size(), True))
if (self.params.selection is not None):
self.top_selection = flex.smart_selection(
flags=asc.selection(self.params.selection))
self._rotate_about_axis()
self._process_adp()
self._process_sites()
self._process_occupancies()
self._put_in_box()
self._change_of_basis()
# Up to this point we are done with self.xray_structure
self.pdb_hierarchy.adopt_xray_structure(self.xray_structure)
# Now only manipulations that use self.pdb_hierarchy are done
if (self.params.set_chemical_element_simple_if_necessary
or self.params.rename_chain_id.old_id
or self.params.renumber_residues
or self.params.increment_resseq
or self.params.convert_semet_to_met
or self.params.convert_met_to_semet
or self.params.set_charge.charge
or self.params.truncate_to_polyala
or self.params.truncate_to_polygly
or self.params.remove_alt_confs or self.params.move_waters_last
or self.params.remove_fraction or self.params.keep
or self.params.remove):
del self.xray_structure # it is invalide below this point
self._set_chemical_element_simple_if_necessary()
self._rename_chain_id()
self._renumber_residues()
self._convert_semet_to_met()
self._convert_met_to_semet()
self._set_atomic_charge()
self._truncate_to_poly_ala()
self._truncate_to_poly_gly()
self._remove_alt_confs()
self._move_waters()
self._remove_atoms()
self._apply_keep_remove()
def exercise_unmerged():
quartz_structure = xray.structure(
special_position_settings=crystal.special_position_settings(
crystal_symmetry=crystal.symmetry(unit_cell=(5.01, 5.01, 5.47, 90,
90, 120),
space_group_symbol="P6222")),
scatterers=flex.xray_scatterer([
xray.scatterer(label="Si", site=(1 / 2., 1 / 2., 1 / 3.), u=0.2),
xray.scatterer(label="O", site=(0.197, -0.197, 0.83333), u=0.1)
]))
quartz_structure.set_inelastic_form_factors(photon=1.54, table="sasaki")
fc = abs(quartz_structure.structure_factors(d_min=1.0).f_calc())
symm = fc.crystal_symmetry()
icalc = fc.expand_to_p1().f_as_f_sq().set_observation_type_xray_intensity()
# generate 'unmerged' data
i_obs = icalc.customized_copy(crystal_symmetry=symm)
# now make up sigmas and some (hopefully realistic) error
flex.set_random_seed(12345)
n_refl = i_obs.size()
sigmas = flex.random_double(n_refl) * flex.mean(fc.data())
sigmas = icalc.customized_copy(data=sigmas).apply_debye_waller_factors(
u_iso=0.15)
err = (flex.double(n_refl, 0.5) - flex.random_double(n_refl)) * 2
i_obs = i_obs.customized_copy(sigmas=sigmas.data(),
data=i_obs.data() + err)
# check for unmerged acentrics
assert i_obs.is_unmerged_intensity_array()
i_obs_centric = i_obs.select(i_obs.centric_flags().data())
i_obs_acentric = i_obs.select(~(i_obs.centric_flags().data()))
i_mrg_acentric = i_obs_acentric.merge_equivalents().array()
i_mixed = i_mrg_acentric.concatenate(i_obs_centric)
assert not i_mixed.is_unmerged_intensity_array()
# XXX These results of these functions are heavily dependent on the
# behavior of the random number generator, which is not consistent across
# platforms - therefore we can only check for very approximate values.
# Exact numerical results are tested with real data (stored elsewhere).
# CC1/2, etc.
assert approx_equal(i_obs.cc_one_half(), 0.9999, eps=0.001)
assert approx_equal(i_obs.cc_one_half_sigma_tau(), 0.9999, eps=0.001)
assert i_obs.resolution_filter(d_max=1.2).cc_one_half() > 0
assert i_obs.cc_anom() > 0.1
r_ano = i_obs.r_anom()
assert approx_equal(r_ano, 0.080756, eps=0.0001)
# merging stats
i_mrg = i_obs.merge_equivalents()
assert i_mrg.r_merge() < 0.1
assert i_mrg.r_meas() < 0.1
assert i_mrg.r_pim() < 0.05
def exercise_split_unmerged():
import random
random.seed(42)
flex.set_random_seed(42)
from cctbx import crystal
base_set = miller.build_set(
crystal_symmetry=crystal.symmetry(
unit_cell=(10,10,10,90,90,90), space_group_symbol="P1"),
d_min=1.6,
anomalous_flag=False)
indices = base_set.indices()
assert (len(indices) == 510)
unmerged_hkl = flex.miller_index()
unmerged_data = flex.double()
unmerged_sigmas = flex.double()
redundancies = flex.size_t()
# XXX grossly overengineered, but I wanted to get a realistic CC to make sure
# the reflections are being split properly
for i, hkl in enumerate(indices):
n_obs = min(8, 1 + i % 12)
redundancies.append(n_obs)
intensity_merged = (510 - i) + (510 % 27)
for j in range(n_obs):
unmerged_hkl.append(hkl)
intensity = intensity_merged + 20 * (510 % (7 * (j+1)))
sigma = max(0.5, i % 10)
unmerged_data.append(intensity)
unmerged_sigmas.append(sigma)
assert (unmerged_hkl.size() == 2877)
unmerged_array = miller.set(
crystal_symmetry=base_set,
indices=unmerged_hkl,
anomalous_flag=False).array(data=unmerged_data, sigmas=unmerged_sigmas)
split = miller.split_unmerged(
unmerged_indices=unmerged_hkl,
unmerged_data=unmerged_data,
unmerged_sigmas=unmerged_sigmas)
assert (split.data_1.size() == split.data_2.size() == 467)
cc = miller.compute_cc_one_half(unmerged_array)
assert approx_equal(cc, 0.861, eps=0.001)
unmerged_array.setup_binner(n_bins=10)
unmerged_array.set_observation_type_xray_intensity()
result = unmerged_array.cc_one_half(use_binning=True)
assert approx_equal(
result.data[1:-1],
[0.549, 0.789, 0.843, 0.835, 0.863, 0.860, 0.893, 0.847, 0.875, 0.859],
eps=0.05)
def exercise_split_unmerged () :
import random
random.seed(42)
flex.set_random_seed(42)
from cctbx import crystal
base_set = miller.build_set(
crystal_symmetry=crystal.symmetry(
unit_cell=(10,10,10,90,90,90), space_group_symbol="P1"),
d_min=1.6,
anomalous_flag=False)
indices = base_set.indices()
assert (len(indices) == 510)
unmerged_hkl = flex.miller_index()
unmerged_data = flex.double()
unmerged_sigmas = flex.double()
redundancies = flex.size_t()
# XXX grossly overengineered, but I wanted to get a realistic CC to make sure
# the reflections are being split properly
for i, hkl in enumerate(indices) :
n_obs = min(8, 1 + i % 12)
redundancies.append(n_obs)
intensity_merged = (510 - i) + (510 % 27)
for j in range(n_obs) :
unmerged_hkl.append(hkl)
intensity = intensity_merged + 20 * (510 % (7 * (j+1)))
sigma = max(0.5, i % 10)
unmerged_data.append(intensity)
unmerged_sigmas.append(sigma)
assert (unmerged_hkl.size() == 2877)
unmerged_array = miller.set(
crystal_symmetry=base_set,
indices=unmerged_hkl,
anomalous_flag=False).array(data=unmerged_data, sigmas=unmerged_sigmas)
split = miller.split_unmerged(
unmerged_indices=unmerged_hkl,
unmerged_data=unmerged_data,
unmerged_sigmas=unmerged_sigmas)
assert (split.data_1.size() == split.data_2.size() == 467)
cc = miller.compute_cc_one_half(unmerged_array)
assert approx_equal(cc, 0.861, eps=0.001)
unmerged_array.setup_binner(n_bins=10)
unmerged_array.set_observation_type_xray_intensity()
result = unmerged_array.cc_one_half(use_binning=True)
assert approx_equal(
result.data[1:-1],
[0.549, 0.789, 0.843, 0.835, 0.863, 0.860, 0.893, 0.847, 0.875, 0.859],
eps=0.05)
def exercise_4_f_hydrogens():
for d_min in [1, 2, 3]:
for q in [0, 0.9, 1]:
random.seed(0)
flex.set_random_seed(0)
x = random_structure.xray_structure(
space_group_info=sgtbx.space_group_info("P 4"),
elements=(("O", "N", "C") * 5 + ("H", ) * 95),
volume_per_atom=200,
min_distance=1.5,
general_positions_only=True,
random_u_iso=True,
random_occupancy=False)
hd_sel = x.hd_selection()
b_isos = x.select(
~hd_sel).extract_u_iso_or_u_equiv() * adptbx.u_as_b(1.)
mmm = b_isos.min_max_mean().as_tuple()
b_mean = int(mmm[2])
x = x.set_b_iso(value=b_mean, selection=hd_sel)
x.scattering_type_registry(table="wk1995")
x.set_occupancies(value=q, selection=hd_sel)
fc = x.structure_factors(d_min=d_min, algorithm="direct").f_calc()
f_obs = abs(fc)
x = x.deep_copy_scatterers()
x.set_occupancies(value=0.0, selection=x.hd_selection())
sfg_params = mmtbx.f_model.sf_and_grads_accuracy_master_params.extract(
)
sfg_params.algorithm = "direct"
r_free_flags = f_obs.generate_r_free_flags(fraction=0.1)
fmodel = mmtbx.f_model.manager(
xray_structure=x,
f_obs=f_obs,
r_free_flags=r_free_flags,
sf_and_grads_accuracy_params=sfg_params)
if (q == 0):
assert approx_equal(fmodel.r_work(), 0)
else:
assert fmodel.r_work() > 0.05, fmodel.r_work()
params = bss.master_params.extract()
params.bulk_solvent = False
params.anisotropic_scaling = False
o = fmodel.update_all_scales(fast=False, params=params)
assert approx_equal(o.k_sol[0], 0)
assert approx_equal(o.b_sol[0], 0)
assert approx_equal(o.b_cart, [0, 0, 0, 0, 0, 0])
assert approx_equal(o.k_h, q)
assert approx_equal(fmodel.r_work(), 0)
def exercise(self,
xray_structure=None,
space_group_info=None,
verbose=False,
fixed_random_seed=True,
**kwds):
assert [xray_structure, space_group_info].count(None) == 1
if xray_structure is None:
self.xs = self.random_structure(space_group_info, set_grads=True)
else:
self.xs = xray_structure
if fixed_random_seed:
random.seed(1)
flex.set_random_seed(1)
self.do_exercise(verbose)
def exercise_4_f_hydrogens():
for d_min in [1,2,3]:
for q in [0, 0.9, 1]:
random.seed(0)
flex.set_random_seed(0)
x = random_structure.xray_structure(
space_group_info = sgtbx.space_group_info("P 4"),
elements =(("O","N","C")*5 + ("H",)*95),
volume_per_atom = 200,
min_distance = 1.5,
general_positions_only = True,
random_u_iso = True,
random_occupancy = False)
hd_sel = x.hd_selection()
b_isos = x.select(~hd_sel).extract_u_iso_or_u_equiv()*adptbx.u_as_b(1.)
mmm = b_isos.min_max_mean().as_tuple()
b_mean = int(mmm[2])
x = x.set_b_iso(value=b_mean, selection = hd_sel)
x.scattering_type_registry(table="wk1995")
x.set_occupancies(value=q, selection = hd_sel)
fc = x.structure_factors(d_min = d_min, algorithm="direct").f_calc()
f_obs = abs(fc)
x = x.deep_copy_scatterers()
x.set_occupancies(value=0.0, selection = x.hd_selection())
sfg_params = mmtbx.f_model.sf_and_grads_accuracy_master_params.extract()
sfg_params.algorithm = "direct"
r_free_flags = f_obs.generate_r_free_flags(fraction = 0.1)
fmodel = mmtbx.f_model.manager(
xray_structure = x,
f_obs = f_obs,
r_free_flags = r_free_flags,
sf_and_grads_accuracy_params = sfg_params)
if(q==0):
assert approx_equal(fmodel.r_work(), 0)
else:
assert fmodel.r_work() > 0.05, fmodel.r_work()
params = bss.master_params.extract()
params.bulk_solvent=False
params.anisotropic_scaling=False
o = fmodel.update_all_scales(fast=False, params=params)
assert approx_equal(o.k_sol[0],0)
assert approx_equal(o.b_sol[0],0)
assert approx_equal(o.b_cart,[0,0,0,0,0,0])
assert approx_equal(o.k_h, q)
assert approx_equal(fmodel.r_work(), 0)
def exercise(args):
forever = False
random_seed = None
for arg in args:
if (arg == "--forever"):
forever = True
elif (arg.startswith("--random_seed=")):
random_seed = int(arg.split("=", 1)[1])
if (random_seed is None):
random_seed = flex.get_random_seed()
while True:
print "random_seed:", random_seed
random.seed(random_seed)
flex.set_random_seed(value=random_seed)
exercise_2()
if (not forever): break
random_seed += 1
print format_cpu_times()
def exercise(args):
forever = False
random_seed = None
for arg in args:
if (arg == "--forever"):
forever = True
elif (arg.startswith("--random_seed=")):
random_seed = int(arg.split("=", 1)[1])
if (random_seed is None):
random_seed = flex.get_random_seed()
while True:
print("random_seed:", random_seed)
random.seed(random_seed)
flex.set_random_seed(value=random_seed)
exercise_2()
if (not forever): break
random_seed += 1
print(format_cpu_times())
def exercise_top_largest_f_obs_f_model_differences(threshold_percent=10,
symbol="C 2"):
random.seed(0)
flex.set_random_seed(0)
x = random_structure.xray_structure(
space_group_info=sgtbx.space_group_info(symbol=symbol),
elements=(("O", "N", "C") * 5 + ("H", ) * 10),
volume_per_atom=200,
min_distance=1.5,
general_positions_only=True,
random_u_iso=True,
random_occupancy=False)
f_obs = abs(x.structure_factors(d_min=1.5, algorithm="fft").f_calc())
x.shake_sites_in_place(mean_distance=1)
fmodel = mmtbx.f_model.manager(xray_structure=x, f_obs=f_obs)
fmodel.update_all_scales(update_f_part1=False)
v, d = fmodel.top_largest_f_obs_f_model_differences(
threshold_percent=threshold_percent)
n = (d > v).count(True) * 100. / d.size()
assert approx_equal(threshold_percent, n, 0.5)
def run():
import libtbx.utils
libtbx.utils.show_times_at_exit()
import sys
from libtbx.option_parser import option_parser
command_line = (option_parser().option(
None, "--normal_eqns_solving_method",
default='naive').option(None,
"--fix_random_seeds",
action='store_true',
default='naive')).process(args=sys.argv[1:])
opts = command_line.options
if opts.fix_random_seeds:
import random
random.seed(1)
flex.set_random_seed(1)
gradient_threshold = 1e-8
step_threshold = 1e-8
if opts.normal_eqns_solving_method == 'naive':
m = lambda eqns: normal_eqns_solving.naive_iterations(
eqns,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold)
elif opts.normal_eqns_solving_method == 'levenberg-marquardt':
m = lambda eqns: normal_eqns_solving.levenberg_marquardt_iterations(
eqns,
gradient_threshold=gradient_threshold,
step_threshold=gradient_threshold,
tau=1e-7)
else:
raise RuntimeError("Unknown method %s" %
opts.normal_eqns_solving_method)
for t in [
saturated_test_case(m),
sucrose_test_case(m),
symmetry_equivalent_test_case(m),
fpfdp_test_case(m),
constrained_fpfdp_test_case(m),
scalar_scaled_adp_test_case(m),
]:
t.run()
def run(args):
if (1):
random.seed(0)
flex.set_random_seed(0)
out = null_out()
remaining_args = []
for arg in args:
if (arg == "--verbose"): out = sys.stdout
else: remaining_args.append(arg)
test_cases = test_cases_tardy_pdb.select_test_cases(
tags_or_indices=remaining_args)
for test_case in test_cases:
print >> out, "test case %d: %s" % (test_case.index, test_case.tag)
minimized = []
for use_geo in [False, True]:
minimized.append(exercise_lbfgs(test_case, use_geo=use_geo, out=out))
m0, m1 = minimized
assert m0.real_space_target_weight == 1
assert m1.real_space_target_weight == 1
assert m1.f_final < m0.f_start * 0.98
print format_cpu_times()
def exercise(flags, space_group_info):
if not flags.repeats:
flags.repeats = 1
if not flags.algo:
flags.algo = "weak_reflection_improved"
if not flags.on:
flags.on = "E"
if flags.fix_seed:
random.seed(1)
flex.set_random_seed(1)
n = len(space_group_info.group())
print space_group_info.type().hall_symbol(),
if not flags.high_symmetry and n > 24:
print " [ skipped ]"
if flags.Verbose:
print
return
else:
print
n_C = 12 // n or 1
n_O = 6 // n
n_N = 3 // n
if flags.Verbose:
print "unit cell content: C%i O%i N%i" % (n_C * n, n_O * n, n_N * n)
print "asu content: C%i O%i N%i" % (n_C, n_O, n_N)
print "on %s's with %s" % (flags.on, flags.algo)
flipping_type = eval("charge_flipping.%s_iterator" % flags.algo)
for i in xrange(int(flags.repeats)):
randomly_exercise(
flipping_type=flipping_type,
space_group_info=space_group_info,
elements=["C"] * n_C + ["O"] * n_O + ["N"] * n_N,
anomalous_flag=False,
d_min=0.8,
verbose=flags.Verbose,
amplitude_type=flags.on,
)
if flags.Verbose:
print
def run(args):
if (1):
random.seed(0)
flex.set_random_seed(0)
out = null_out()
remaining_args = []
for arg in args:
if (arg == "--verbose"): out = sys.stdout
else: remaining_args.append(arg)
test_cases = test_cases_tardy_pdb.select_test_cases(
tags_or_indices=remaining_args)
for test_case in test_cases:
print("test case %d: %s" % (test_case.index, test_case.tag), file=out)
minimized = []
for use_geo in [False, True]:
minimized.append(
exercise_lbfgs(test_case, use_geo=use_geo, out=out))
m0, m1 = minimized
assert m0.real_space_target_weight == 1
assert m1.real_space_target_weight == 1
assert m1.f_final < m0.f_start * 0.98
print(format_cpu_times())
def exercise_top_largest_f_obs_f_model_differences(threshold_percent=10,
symbol = "C 2"):
random.seed(0)
flex.set_random_seed(0)
x = random_structure.xray_structure(
space_group_info = sgtbx.space_group_info(symbol=symbol),
elements =(("O","N","C")*5+("H",)*10),
volume_per_atom = 200,
min_distance = 1.5,
general_positions_only = True,
random_u_iso = True,
random_occupancy = False)
f_obs = abs(x.structure_factors(d_min = 1.5, algorithm="fft").f_calc())
x.shake_sites_in_place(mean_distance=1)
fmodel = mmtbx.f_model.manager(
xray_structure = x,
f_obs = f_obs)
fmodel.update_all_scales(update_f_part1=False)
v, d = fmodel.top_largest_f_obs_f_model_differences(
threshold_percent=threshold_percent)
n = (d>v).count(True)*100./d.size()
assert approx_equal(threshold_percent, n, 0.5)
def run():
import libtbx.utils
libtbx.utils.show_times_at_exit()
import sys
from libtbx.option_parser import option_parser
command_line = (option_parser()
.option(None, "--normal_eqns_solving_method",
default='naive')
.option(None, "--fix_random_seeds",
action='store_true',
default='naive')
).process(args=sys.argv[1:])
opts = command_line.options
if opts.fix_random_seeds:
import random
random.seed(1)
flex.set_random_seed(1)
gradient_threshold=1e-8
step_threshold=1e-8
if opts.normal_eqns_solving_method == 'naive':
m = lambda eqns: normal_eqns_solving.naive_iterations(
eqns,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold)
elif opts.normal_eqns_solving_method == 'levenberg-marquardt':
m = lambda eqns: normal_eqns_solving.levenberg_marquardt_iterations(
eqns,
gradient_threshold=gradient_threshold,
step_threshold=gradient_threshold,
tau=1e-7)
else:
raise RuntimeError("Unknown method %s" % opts.normal_eqns_solving_method)
for t in [
saturated_test_case(m),
sucrose_test_case(m),
symmetry_equivalent_test_case(m),
]:
t.run()
def run(args):
master_phil = libtbx.phil.parse("""
d_min = 0.5
.type = float
constrained_refinement = True
.type = bool
random_seed = 1
.type = int
shake_sites_rmsd = 0.5
.type = float
shake_adp_spread = 20
.type = float
grad_site=True
.type = bool
grad_u_iso=False
.type = bool
grad_u_aniso=False
.type = bool
grad_occupancy=False
.type = bool
grad_fp_fdp=False
.type = bool
lbfgs_m = 5
.type = int
lbfgs_max_iterations = 1000
.type = int
verbose = 0
.type = int
""")
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
params = work_phil.extract()
if params.random_seed is not None:
import scitbx.random
import random
scitbx.random.set_random_seed(params.random_seed)
flex.set_random_seed(params.random_seed)
random.seed(params.random_seed)
if len(remaining_args):
assert len(remaining_args) == 1
file_path = remaining_args[0]
root, ext = os.path.splitext(file_path)
if ext == ".cif":
xs_dict = xray.structure.from_cif(file_path=file_path)
assert len(
xs_dict) == 1, "CIF should contain only one xray structure"
xs = xs_dict.values()[0]
xs.show_summary().show_scatterers()
print
constraints_list = None
t_celsius = 20
else:
raise RuntimeError("Only CIF format currently supported!")
else:
test_case = tst_constrained_structure.sucrose_test_case(None)
t_celsius = test_case.t_celsius
xs = test_case.xray_structure
constraints_list = test_case.constraints
#from cctbx import adptbx
#for sc in xs.scatterers():
#if sc.flags.use_u_aniso():
#sc.u_iso = adptbx.u_star_as_u_iso(xs.unit_cell(), sc.u_star)
#sc.set_use_u_iso_only()
if not params.constrained_refinement:
constraints_list = []
exercise_constrained_lbfgs(
xray_structure=xs,
constraints_list=constraints_list,
t_celsius=t_celsius,
d_min=params.d_min,
shake_sites_rmsd=params.shake_sites_rmsd,
shake_u_iso_spread=params.shake_adp_spread,
grad_site=params.grad_site,
grad_u_iso=params.grad_u_iso,
grad_u_aniso=params.grad_u_aniso,
grad_occupancy=params.grad_occupancy,
grad_fp_fdp=params.grad_fp_fdp,
lbfgs_m=params.lbfgs_m,
lbfgs_max_iterations=params.lbfgs_max_iterations,
verbose=params.verbose)
def exercise_cif_from_cctbx():
quartz = xray.structure(
crystal_symmetry=crystal.symmetry(
(5.01,5.01,5.47,90,90,120), "P6222"),
scatterers=flex.xray_scatterer([
xray.scatterer("Si", (1/2.,1/2.,1/3.)),
xray.scatterer("O", (0.197,-0.197,0.83333))]))
for sc in quartz.scatterers():
sc.flags.set_grad_site(True)
s = StringIO()
loop = geometry.distances_as_cif_loop(
quartz.pair_asu_table(distance_cutoff=2),
site_labels=quartz.scatterers().extract_labels(),
sites_frac=quartz.sites_frac()).loop
print >> s, loop
assert not show_diff(s.getvalue(), """\
loop_
_geom_bond_atom_site_label_1
_geom_bond_atom_site_label_2
_geom_bond_distance
_geom_bond_site_symmetry_2
Si O 1.6160 4_554
Si O 1.6160 2_554
Si O 1.6160 3_664
Si O 1.6160 5_664
""")
s = StringIO()
loop = geometry.angles_as_cif_loop(
quartz.pair_asu_table(distance_cutoff=2),
site_labels=quartz.scatterers().extract_labels(),
sites_frac=quartz.sites_frac()).loop
print >> s, loop
assert not show_diff(s.getvalue(), """\
loop_
_geom_angle_atom_site_label_1
_geom_angle_atom_site_label_2
_geom_angle_atom_site_label_3
_geom_angle
_geom_angle_site_symmetry_1
_geom_angle_site_symmetry_3
O Si O 101.3 2_554 4_554
O Si O 111.3 3_664 4_554
O Si O 116.1 3_664 2_554
O Si O 116.1 5_664 4_554
O Si O 111.3 5_664 2_554
O Si O 101.3 5_664 3_664
Si O Si 146.9 3 5
""")
# with a covariance matrix
flex.set_random_seed(1)
vcv_matrix = matrix.diag(
flex.random_double(size=quartz.n_parameters(), factor=1e-5))\
.as_flex_double_matrix().matrix_symmetric_as_packed_u()
s = StringIO()
loop = geometry.distances_as_cif_loop(
quartz.pair_asu_table(distance_cutoff=2),
site_labels=quartz.scatterers().extract_labels(),
sites_frac=quartz.sites_frac(),
covariance_matrix=vcv_matrix,
parameter_map=quartz.parameter_map()).loop
print >> s, loop
assert not show_diff(s.getvalue(), """\
loop_
_geom_bond_atom_site_label_1
_geom_bond_atom_site_label_2
_geom_bond_distance
_geom_bond_site_symmetry_2
Si O 1.616(14) 4_554
Si O 1.616(12) 2_554
Si O 1.616(14) 3_664
Si O 1.616(12) 5_664
""")
s = StringIO()
loop = geometry.angles_as_cif_loop(
quartz.pair_asu_table(distance_cutoff=2),
site_labels=quartz.scatterers().extract_labels(),
sites_frac=quartz.sites_frac(),
covariance_matrix=vcv_matrix,
parameter_map=quartz.parameter_map()).loop
print >> s, loop
assert not show_diff(s.getvalue(), """\
loop_
_geom_angle_atom_site_label_1
_geom_angle_atom_site_label_2
_geom_angle_atom_site_label_3
_geom_angle
_geom_angle_site_symmetry_1
_geom_angle_site_symmetry_3
O Si O 101.3(8) 2_554 4_554
O Si O 111.3(10) 3_664 4_554
O Si O 116.1(9) 3_664 2_554
O Si O 116.1(9) 5_664 4_554
O Si O 111.3(10) 5_664 2_554
O Si O 101.3(8) 5_664 3_664
Si O Si 146.9(9) 3 5
""")
cell_vcv = flex.pow2(matrix.diag(flex.random_double(size=6,factor=1e-1))\
.as_flex_double_matrix().matrix_symmetric_as_packed_u())
s = StringIO()
loop = geometry.distances_as_cif_loop(
quartz.pair_asu_table(distance_cutoff=2),
site_labels=quartz.scatterers().extract_labels(),
sites_frac=quartz.sites_frac(),
covariance_matrix=vcv_matrix,
cell_covariance_matrix=cell_vcv,
parameter_map=quartz.parameter_map()).loop
print >> s, loop
assert not show_diff(s.getvalue(), """\
loop_
_geom_bond_atom_site_label_1
_geom_bond_atom_site_label_2
_geom_bond_distance
_geom_bond_site_symmetry_2
Si O 1.616(15) 4_554
Si O 1.616(19) 2_554
Si O 1.616(15) 3_664
Si O 1.616(19) 5_664
""")
def run(args):
from dials.util import log
usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
phil=phil_scope,
read_datablocks=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=False)
datablocks = flatten_datablocks(params.input.datablock)
reflections = flatten_reflections(params.input.reflections)
if len(datablocks) == 0 or len(reflections) == 0:
parser.print_help()
exit(0)
# Configure the logging
log.config(info=params.output.log, debug=params.output.debug_log)
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
if params.seed is not None:
import random
flex.set_random_seed(params.seed)
random.seed(params.seed)
imagesets = []
for datablock in datablocks:
imagesets.extend(datablock.extract_imagesets())
assert len(imagesets) > 0
assert len(reflections) == len(imagesets)
if params.scan_range is not None and len(params.scan_range) > 0:
reflections = [
filter_reflections_by_scan_range(refl, params.scan_range)
for refl in reflections
]
dps_params = dps_phil_scope.extract()
# for development, we want an exhaustive plot of beam probability map:
dps_params.indexing.plot_search_scope = params.plot_search_scope
dps_params.indexing.mm_search_scope = params.mm_search_scope
for i in range(params.n_macro_cycles):
if params.n_macro_cycles > 1:
logger.info('Starting macro cycle %i' % (i + 1))
new_detector, new_beam = discover_better_experimental_model(
imagesets,
reflections,
params,
dps_params,
nproc=params.nproc,
wide_search_binning=params.wide_search_binning)
for imageset in imagesets:
imageset.set_detector(new_detector)
imageset.set_beam(new_beam)
logger.info('')
from dxtbx.serialize import dump
logger.info("Saving optimized datablock to %s" % params.output.datablock)
dump.datablock(datablock, params.output.datablock)
def try_den_weight_cartesian(self, grid_pair):
local_seed = int(self.random_seed+grid_pair[1])
flex.set_random_seed(value=local_seed)
random.seed(local_seed)
self.fmodels.fmodel_xray().xray_structure.replace_scatterers(
self.save_scatterers_local.deep_copy())
self.fmodels.update_xray_structure(
xray_structure = self.fmodels.fmodel_xray().xray_structure,
update_f_calc = True)
utils.assert_xray_structures_equal(
x1 = self.fmodels.fmodel_xray().xray_structure,
x2 = self.model.get_xray_structure())
gamma_local = grid_pair[0]
weight_local = grid_pair[1]
self.model.restraints_manager.geometry.\
den_manager.gamma = gamma_local
self.model.restraints_manager.geometry.\
den_manager.weight = weight_local
cycle = 0
self.model.restraints_manager.geometry.\
den_manager.current_cycle = cycle+1
num_den_cycles = self.model.restraints_manager.geometry.\
den_manager.num_cycles
if self.params.den.optimize and \
self.nproc != Auto and \
self.nproc > 1:
local_log = sys.stdout
elif self.params.den.optimize and \
self.nproc == 1:
if self.verbose:
local_log = self.log
else:
local_log = StringIO()
else:
local_log = self.log
print >> self.log, " ...trying gamma %f, weight %f" % (
gamma_local, weight_local)
while cycle < num_den_cycles:
print >> local_log, "DEN cycle %s" % (cycle+1)
r_free = self.fmodels.fmodel_xray().r_free()
print >> local_log, "rfree at start of SA cycle: %f" % r_free
simulated_annealing.manager(
params = self.params.simulated_annealing,
target_weights = self.target_weights,
macro_cycle = self.macro_cycle,
h_params = self.params.hydrogens,
fmodels = self.fmodels,
model = self.model,
all_params = self.params,
out = local_log)
if self.params.den.bulk_solvent_and_scale:
self.bulk_solvent_and_scale(log=local_log)
if self.params.den.refine_adp:
self.adp_refinement(log=local_log)
self.model.torsion_ncs_restraints_update(log=local_log)
cycle += 1
self.model.restraints_manager.geometry.\
den_manager.current_cycle += 1
r_free = self.fmodels.fmodel_xray().r_free()
print >> local_log, "rfree at end of SA cycle: %f" % r_free
r_free = self.fmodels.fmodel_xray().r_free()
step_xray_structure = self.fmodels.fmodel_xray().\
xray_structure.deep_copy_scatterers().scatterers()
step_eq_distances = self.model.restraints_manager.geometry.\
den_manager.get_current_eq_distances()
return (gamma_local,
weight_local,
r_free,
step_xray_structure,
step_eq_distances)
def run(
args,
command_name="phenix.reflection_file_converter",
simply_return_all_miller_arrays=False):
command_line = (option_parser(
usage="%s [options] reflection_file ..." % command_name,
description="Example: %s w1.sca --mtz ." % command_name)
.enable_symmetry_comprehensive()
.option(None, "--weak_symmetry",
action="store_true",
default=False,
help="symmetry on command line is weaker than symmetry found in files")
.enable_resolutions()
.option(None, "--label",
action="store",
type="string",
help="Substring of reflection data label or number",
metavar="STRING")
.option(None, "--non_anomalous",
action="store_true",
default=False,
help="Averages Bijvoet mates to obtain a non-anomalous array")
.option(None, "--r_free_label",
action="store",
type="string",
help="Substring of reflection data label or number",
metavar="STRING")
.option(None, "--r_free_test_flag_value",
action="store",
type="int",
help="Value in R-free array indicating assignment to free set.",
metavar="FLOAT")
.option(None, "--generate_r_free_flags",
action="store_true",
default=False,
help="Generates a new array of random R-free flags"
" (MTZ and CNS output only).")
.option(None, "--use_lattice_symmetry_in_r_free_flag_generation",
dest="use_lattice_symmetry_in_r_free_flag_generation",
action="store_true",
default=True,
help="group twin/pseudo symmetry related reflections together"
" in r-free set (this is the default).")
.option(None, "--no_lattice_symmetry_in_r_free_flag_generation",
dest="use_lattice_symmetry_in_r_free_flag_generation",
action="store_false",
help="opposite of --use-lattice-symmetry-in-r-free-flag-generation")
.option(None, "--r_free_flags_fraction",
action="store",
default=0.10,
type="float",
help="Target fraction free/work reflections (default: 0.10).",
metavar="FLOAT")
.option(None, "--r_free_flags_max_free",
action="store",
default=2000,
type="int",
help="Maximum number of free reflections (default: 2000).",
metavar="INT")
.option(None, "--r_free_flags_format",
choices=("cns", "ccp4", "shelx"),
default="cns",
help="Convention for generating R-free flags",
metavar="cns|ccp4")
.option(None, "--output_r_free_label",
action="store",
type="string",
help="Label for newly generated R-free flags (defaults to R-free-flags)",
default="R-free-flags",
metavar="STRING")
.option(None, "--random_seed",
action="store",
type="int",
help="Seed for random number generator (affects generation of"
" R-free flags).",
metavar="INT")
.option(None, "--change_of_basis",
action="store",
type="string",
help="Change-of-basis operator: h,k,l or x,y,z"
" or to_reference_setting, to_primitive_setting, to_niggli_cell,"
" to_inverse_hand",
metavar="STRING")
.option(None, "--eliminate_invalid_indices",
action="store_true",
default=False,
help="Remove indices which are invalid given the change of basis desired")
.option(None, "--expand_to_p1",
action="store_true",
default=False,
help="Generates all symmetrically equivalent reflections."
" The space group symmetry is reset to P1."
" May be used in combination with --change_to_space_group to"
" lower the symmetry.")
.option(None, "--change_to_space_group",
action="store",
type="string",
help="Changes the space group and merges equivalent reflections"
" if necessary",
metavar="SYMBOL|NUMBER")
.option(None, "--write_mtz_amplitudes",
action="store_true",
default=False,
help="Converts intensities to amplitudes before writing MTZ format;"
" requires --mtz_root_label")
.option(None, "--write_mtz_intensities",
action="store_true",
default=False,
help="Converts amplitudes to intensities before writing MTZ format;"
" requires --mtz_root_label")
.option(None,"--remove_negatives",
action="store_true",
default=False,
help="Remove negative intensities or amplitudes from the data set" )
.option(None,"--massage_intensities",
action="store_true",
default=False,
help="'Treat' negative intensities to get a positive amplitude."
" |Fnew| = sqrt((Io+sqrt(Io**2 +2sigma**2))/2.0). Requires"
" intensities as input and the flags --mtz,"
" --write_mtz_amplitudes and --mtz_root_label.")
.option(None, "--scale_max",
action="store",
type="float",
help="Scales data such that the maximum is equal to the given value",
metavar="FLOAT")
.option(None, "--scale_factor",
action="store",
type="float",
help="Multiplies data with the given factor",
metavar="FLOAT")
.option(None, "--sca",
action="store",
type="string",
help=
"write data to Scalepack FILE ('--sca .' copies name of input file)",
metavar="FILE")
.option(None, "--mtz",
action="store",
type="string",
help="write data to MTZ FILE ('--mtz .' copies name of input file)",
metavar="FILE")
.option(None, "--mtz_root_label",
action="store",
type="string",
help="Root label for MTZ file (e.g. Fobs)",
metavar="STRING")
.option(None, "--cns",
action="store",
type="string",
help="write data to CNS FILE ('--cns .' copies name of input file)",
metavar="FILE")
.option(None, "--shelx",
action="store",
type="string",
help="write data to SHELX FILE ('--shelx .' copies name of input file)",
metavar="FILE")
).process(args=args)
co = command_line.options
if (co.random_seed is not None):
random.seed(co.random_seed)
flex.set_random_seed(value=co.random_seed)
if ( co.write_mtz_amplitudes
and co.write_mtz_intensities):
print
print "--write_mtz_amplitudes and --write_mtz_intensities" \
" are mutually exclusive."
print
return None
if ( co.write_mtz_amplitudes
or co.write_mtz_intensities):
if (co.mtz_root_label is None):
print
print "--write_mtz_amplitudes and --write_mtz_intensities" \
" require --mtz_root_label."
print
return None
if ( co.scale_max is not None
and co.scale_factor is not None):
print
print "--scale_max and --scale_factor are mutually exclusive."
print
return None
if (len(command_line.args) == 0):
command_line.parser.show_help()
return None
all_miller_arrays = reflection_file_reader.collect_arrays(
file_names=command_line.args,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=False,
discard_arrays=False,
verbose=1)
if (simply_return_all_miller_arrays):
return all_miller_arrays
if (len(all_miller_arrays) == 0):
print
print "No reflection data found in input file%s." % (
plural_s(len(command_line.args))[1])
print
return None
label_table = reflection_file_utils.label_table(
miller_arrays=all_miller_arrays)
selected_array = label_table.select_array(
label=co.label, command_line_switch="--label")
if (selected_array is None): return None
r_free_flags = None
r_free_info = None
if (co.r_free_label is not None):
r_free_flags = label_table.match_data_label(
label=co.r_free_label,
command_line_switch="--r_free_label")
if (r_free_flags is None):
return None
r_free_info = str(r_free_flags.info())
if (not r_free_flags.is_bool_array()):
test_flag_value = reflection_file_utils.get_r_free_flags_scores(
miller_arrays=[r_free_flags],
test_flag_value=co.r_free_test_flag_value).test_flag_values[0]
if (test_flag_value is None):
if (co.r_free_test_flag_value is None):
raise Sorry(
"Cannot automatically determine r_free_test_flag_value."
" Please use --r_free_test_flag_value to specify a value.")
else:
raise Sorry("Invalid --r_free_test_flag_value.")
r_free_flags = r_free_flags.customized_copy(
data=(r_free_flags.data() == test_flag_value))
print "Selected data:"
print " ", selected_array.info()
print " Observation type:", selected_array.observation_type()
print
if (r_free_info is not None):
print "R-free flags:"
print " ", r_free_info
print
processed_array = selected_array.customized_copy(
crystal_symmetry=selected_array.join_symmetry(
other_symmetry=command_line.symmetry,
force=not co.weak_symmetry)).set_observation_type(
selected_array.observation_type())
if (r_free_flags is not None):
r_free_flags = r_free_flags.customized_copy(
crystal_symmetry=processed_array)
print "Input crystal symmetry:"
crystal.symmetry.show_summary(processed_array, prefix=" ")
print
if (processed_array.unit_cell() is None):
command_line.parser.show_help()
print "Unit cell parameters unknown. Please use --symmetry or --unit_cell."
print
return None
if (processed_array.space_group_info() is None):
command_line.parser.show_help()
print "Space group unknown. Please use --symmetry or --space_group."
print
return None
if (r_free_flags is not None):
r_free_flags = r_free_flags.customized_copy(
crystal_symmetry=processed_array)
if (co.change_of_basis is not None):
processed_array, cb_op = processed_array.apply_change_of_basis(
change_of_basis=co.change_of_basis,
eliminate_invalid_indices=co.eliminate_invalid_indices)
if (r_free_flags is not None):
r_free_flags = r_free_flags.change_basis(cb_op=cb_op)
if (not processed_array.is_unique_set_under_symmetry()):
print "Merging symmetry-equivalent values:"
merged = processed_array.merge_equivalents()
merged.show_summary(prefix=" ")
print
processed_array = merged.array()
del merged
processed_array.show_comprehensive_summary(prefix=" ")
print
if (r_free_flags is not None
and not r_free_flags.is_unique_set_under_symmetry()):
print "Merging symmetry-equivalent R-free flags:"
merged = r_free_flags.merge_equivalents()
merged.show_summary(prefix=" ")
print
r_free_flags = merged.array()
del merged
r_free_flags.show_comprehensive_summary(prefix=" ")
print
if (co.expand_to_p1):
print "Expanding symmetry and resetting space group to P1:"
if (r_free_flags is not None):
raise Sorry(
"--expand_to_p1 not supported for arrays of R-free flags.")
processed_array = processed_array.expand_to_p1()
processed_array.show_comprehensive_summary(prefix=" ")
print
if (co.change_to_space_group is not None):
if (r_free_flags is not None):
raise Sorry(
"--change_to_space_group not supported for arrays of R-free flags.")
new_space_group_info = sgtbx.space_group_info(
symbol=co.change_to_space_group)
print "Change to space group:", new_space_group_info
new_crystal_symmetry = crystal.symmetry(
unit_cell=processed_array.unit_cell(),
space_group_info=new_space_group_info,
assert_is_compatible_unit_cell=False)
if (not new_crystal_symmetry.unit_cell()
.is_similar_to(processed_array.unit_cell())):
print " *************"
print " W A R N I N G"
print " *************"
print " Unit cell parameters adapted to new space group symmetry are"
print " significantly different from input unit cell parameters:"
print " Input unit cell parameters:", \
processed_array.unit_cell()
print " Adapted unit cell parameters:", \
new_crystal_symmetry.unit_cell()
processed_array = processed_array.customized_copy(
crystal_symmetry=new_crystal_symmetry)
print
if (not processed_array.is_unique_set_under_symmetry()):
print " Merging values symmetry-equivalent under new symmetry:"
merged = processed_array.merge_equivalents()
merged.show_summary(prefix=" ")
print
processed_array = merged.array()
del merged
processed_array.show_comprehensive_summary(prefix=" ")
print
if (processed_array.anomalous_flag() and co.non_anomalous):
print "Converting data array from anomalous to non-anomalous."
if (not processed_array.is_xray_intensity_array()):
processed_array = processed_array.average_bijvoet_mates()
else:
processed_array = processed_array.f_sq_as_f()
processed_array = processed_array.average_bijvoet_mates()
processed_array = processed_array.f_as_f_sq()
processed_array.set_observation_type_xray_intensity()
if (r_free_flags is not None
and r_free_flags.anomalous_flag()
and co.non_anomalous):
print "Converting R-free flags from anomalous to non-anomalous."
r_free_flags = r_free_flags.average_bijvoet_mates()
d_max = co.low_resolution
d_min = co.resolution
if (d_max is not None or d_min is not None):
if (d_max is not None):
print "Applying low resolution cutoff: d_max=%.6g" % d_max
if (d_min is not None):
print "Applying high resolution cutoff: d_min=%.6g" % d_min
processed_array = processed_array.resolution_filter(
d_max=d_max, d_min=d_min)
print "Number of reflections:", processed_array.indices().size()
print
if (co.scale_max is not None):
print "Scaling data such that the maximum value is: %.6g" % co.scale_max
processed_array = processed_array.apply_scaling(target_max=co.scale_max)
print
if (co.scale_factor is not None):
print "Multiplying data with the factor: %.6g" % co.scale_factor
processed_array = processed_array.apply_scaling(factor=co.scale_factor)
print
if (([co.remove_negatives, co.massage_intensities]).count(True) == 2):
raise Sorry(
"It is not possible to use --remove_negatives and"
" --massage_intensities at the same time.")
if (co.remove_negatives):
if processed_array.is_real_array():
print "Removing negatives items"
processed_array = processed_array.select(
processed_array.data() > 0)
if processed_array.sigmas() is not None:
processed_array = processed_array.select(
processed_array.sigmas() > 0)
else:
raise Sorry("--remove_negatives not applicable to complex data arrays.")
if (co.massage_intensities):
if processed_array.is_real_array():
if processed_array.is_xray_intensity_array():
if (co.mtz is not None):
if (co.write_mtz_amplitudes):
print "The supplied intensities will be used to estimate"
print " amplitudes in the following way: "
print " Fobs = Sqrt[ (Iobs + Sqrt(Iobs**2 + 2sigmaIobs**2))/2 ]"
print " Sigmas are estimated in a similar manner."
print
processed_array = processed_array.enforce_positive_amplitudes()
else:
raise Sorry(
"--write_mtz_amplitudes has to be specified when using"
" --massage_intensities")
else:
raise Sorry("--mtz has to be used when using --massage_intensities")
else:
raise Sorry(
"Intensities must be supplied when using the option"
" --massage_intensities")
else:
raise Sorry(
"--massage_intensities not applicable to complex data arrays.")
if (not co.generate_r_free_flags):
if (r_free_flags is None):
r_free_info = []
else:
if (r_free_flags.anomalous_flag() != processed_array.anomalous_flag()):
if (processed_array.anomalous_flag()): is_not = ("", " not")
else: is_not = (" not", "")
raise Sorry(
"The data array is%s anomalous but the R-free array is%s.\n"
% is_not
+ " Please try --non_anomalous.")
r_free_info = ["R-free flags source: " + r_free_info]
if (not r_free_flags.indices().all_eq(processed_array.indices())):
processed_array = processed_array.map_to_asu()
r_free_flags = r_free_flags.map_to_asu().common_set(processed_array)
n_missing_r_free_flags = processed_array.indices().size() \
- r_free_flags.indices().size()
if (n_missing_r_free_flags != 0):
raise Sorry("R-free flags not compatible with data array:"
" missing flag for %d reflections selected for output." %
n_missing_r_free_flags)
else:
if (r_free_flags is not None):
raise Sorry(
"--r_free_label and --generate_r_free_flags are mutually exclusive.")
print "Generating a new array of R-free flags:"
r_free_flags = processed_array.generate_r_free_flags(
fraction=co.r_free_flags_fraction,
max_free=co.r_free_flags_max_free,
use_lattice_symmetry=co.use_lattice_symmetry_in_r_free_flag_generation,
format=co.r_free_flags_format)
test_flag_value = True
if (co.r_free_flags_format == "ccp4") :
test_flag_value = 0
elif (co.r_free_flags_format == "shelx") :
test_flag_value = -1
r_free_as_bool = r_free_flags.customized_copy(
data=r_free_flags.data()==test_flag_value)
r_free_info = [
"R-free flags generated by %s:" % command_name]
r_free_info.append(" "+date_and_time())
r_free_info.append(" fraction: %.6g" % co.r_free_flags_fraction)
r_free_info.append(" max_free: %s" % str(co.r_free_flags_max_free))
r_free_info.append(" size of work set: %d" %
r_free_as_bool.data().count(False))
r_free_info.append(" size of free set: %d" %
r_free_as_bool.data().count(True))
r_free_info_str = StringIO()
r_free_as_bool.show_r_free_flags_info(prefix=" ", out=r_free_info_str)
if (co.r_free_flags_format == "ccp4") :
r_free_info.append(" convention: CCP4 (test=0, work=1-%d)" %
flex.max(r_free_flags.data()))
elif (co.r_free_flags_format == "shelx") :
r_free_info.append(" convention: SHELXL (test=-1, work=1)")
else :
r_free_info.append(" convention: CNS/X-PLOR (test=1, work=0)")
print "\n".join(r_free_info[2:4])
print r_free_info[-1]
print r_free_info_str.getvalue()
print
n_output_files = 0
if (co.sca is not None):
if (co.generate_r_free_flags):
raise Sorry("Cannot write R-free flags to Scalepack file.")
file_name = reflection_file_utils.construct_output_file_name(
input_file_names=[selected_array.info().source],
user_file_name=co.sca,
file_type_label="Scalepack",
file_extension="sca")
print "Writing Scalepack file:", file_name
iotbx.scalepack.merge.write(
file_name=file_name,
miller_array=processed_array)
n_output_files += 1
print
if (co.mtz is not None):
file_name = reflection_file_utils.construct_output_file_name(
input_file_names=[selected_array.info().source],
user_file_name=co.mtz,
file_type_label="MTZ",
file_extension="mtz")
print "Writing MTZ file:", file_name
mtz_history_buffer = flex.std_string()
mtz_history_buffer.append(date_and_time())
mtz_history_buffer.append("> program: %s" % command_name)
mtz_history_buffer.append("> input file name: %s" %
os.path.basename(selected_array.info().source))
mtz_history_buffer.append("> input directory: %s" %
os.path.dirname(os.path.abspath(selected_array.info().source)))
mtz_history_buffer.append("> input labels: %s" %
selected_array.info().label_string())
mtz_output_array = processed_array
if (co.write_mtz_amplitudes):
if (not mtz_output_array.is_xray_amplitude_array()):
print " Converting intensities to amplitudes."
mtz_output_array = mtz_output_array.f_sq_as_f()
mtz_history_buffer.append("> Intensities converted to amplitudes.")
elif (co.write_mtz_intensities):
if (not mtz_output_array.is_xray_intensity_array()):
print " Converting amplitudes to intensities."
mtz_output_array = mtz_output_array.f_as_f_sq()
mtz_history_buffer.append("> Amplitudes converted to intensities.")
column_root_label = co.mtz_root_label
if (column_root_label is None):
# XXX 2013-03-29: preserve original root label by default
# XXX 2014-12-16: skip trailing "(+)" in root_label if anomalous
column_root_label = selected_array.info().labels[0]
column_root_label=remove_anomalous_suffix_if_necessary(
miller_array=selected_array,
column_root_label=column_root_label)
mtz_dataset = mtz_output_array.as_mtz_dataset(
column_root_label=column_root_label)
del mtz_output_array
if (r_free_flags is not None):
mtz_dataset.add_miller_array(
miller_array=r_free_flags,
column_root_label=co.output_r_free_label)
for line in r_free_info:
mtz_history_buffer.append("> " + line)
mtz_history_buffer.append("> output file name: %s" %
os.path.basename(file_name))
mtz_history_buffer.append("> output directory: %s" %
os.path.dirname(os.path.abspath(file_name)))
mtz_object = mtz_dataset.mtz_object()
mtz_object.add_history(mtz_history_buffer)
mtz_object.write(file_name=file_name)
n_output_files += 1
print
if (co.cns is not None):
file_name = reflection_file_utils.construct_output_file_name(
input_file_names=[selected_array.info().source],
user_file_name=co.cns,
file_type_label="CNS",
file_extension="cns")
print "Writing CNS file:", file_name
processed_array.export_as_cns_hkl(
file_object=open(file_name, "w"),
file_name=file_name,
info=["source of data: "+str(selected_array.info())] + r_free_info,
r_free_flags=r_free_flags)
n_output_files += 1
print
if (co.shelx is not None):
if (co.generate_r_free_flags):
raise Sorry("Cannot write R-free flags to SHELX file.")
file_name = reflection_file_utils.construct_output_file_name(
input_file_names=[selected_array.info().source],
user_file_name=co.shelx,
file_type_label="SHELX",
file_extension="shelx")
print "Writing SHELX file:", file_name
processed_array.as_amplitude_array().export_as_shelx_hklf(
open(file_name, "w"))
n_output_files += 1
print
if (n_output_files == 0):
command_line.parser.show_help()
print "Please specify at least one output file format,",
print "e.g. --mtz, --sca, etc."
print
return None
return processed_array
def run(args):
import cctbx.omz.dev
import iotbx.cif
import cctbx.xray
#
import random
random.seed(0)
from cctbx.array_family import flex
flex.set_random_seed(0)
#
master_phil = cctbx.omz.dev.get_master_phil(
iteration_limit=100,
additional_phil_string="""\
remove_hydrogen = True
.type = bool
f_obs_is_f_calc = True
.type = bool
reset_u_iso = None
.type = float
""")
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
params = work_phil.extract()
print
#
assert len(remaining_args) == 2, "refl_cif.hkl model.cif"
refl_file = remaining_args[0]
model_file = remaining_args[1]
refl_cif = iotbx.cif.reader(file_path=refl_file)
model_cif = iotbx.cif.reader(file_path=model_file)
#
mgr = extract_from_cif_files(
report_id="cif_refine",
refl_file=refl_file,
refl_cif=refl_cif,
model_file=model_file,
model_cif=model_cif)
#
structure_ideal = mgr.xray_structure.deep_copy_scatterers()
structure_ideal.convert_to_isotropic()
if (params.remove_hydrogen):
sel = mgr.non_hydrogen_selection
print "Removing hydrogen atoms:", sel.count(False)
structure_ideal = structure_ideal.select(selection=sel)
print
#
c_obs = mgr.c_obs
if (params.f_obs_is_f_calc):
c_obs = c_obs.structure_factors_from_scatterers(
xray_structure=structure_ideal,
algorithm="direct",
cos_sin_table=False).f_calc().amplitudes() \
.set_observation_type_xray_amplitude()
#
assert c_obs.is_xray_intensity_array() or c_obs.is_xray_amplitude_array()
if (c_obs.is_xray_intensity_array()):
i_obs = c_obs
f_obs = c_obs.f_sq_as_f(algorithm="xtal_3_7")
else:
f_obs = c_obs
i_obs = c_obs.f_as_f_sq(algorithm="shelxl")
#
structure_shake = structure_ideal.deep_copy_scatterers()
structure_shake.shake_sites_in_place(rms_difference=params.shake_sites_rmsd)
if (params.reset_u_iso is None):
structure_shake.shake_adp(spread=params.shake_adp_spread)
else:
structure_shake.set_u_iso(value=params.reset_u_iso)
#
cctbx.omz.dev.run_refinement(
structure_ideal=structure_ideal,
structure_shake=structure_shake,
params=params,
f_obs=f_obs,
i_obs=i_obs)
def try_den_weight_torsion(self, grid_pair):
#backup_k_rep = self.params.tardy.\
# prolsq_repulsion_function_changes.k_rep
local_seed = int(self.random_seed+grid_pair[1])
flex.set_random_seed(value=local_seed)
random.seed(local_seed)
self.fmodels.fmodel_xray().xray_structure.replace_scatterers(
self.save_scatterers_local.deep_copy())
self.fmodels.update_xray_structure(
xray_structure = self.fmodels.fmodel_xray().xray_structure,
update_f_calc = True)
utils.assert_xray_structures_equal(
x1 = self.fmodels.fmodel_xray().xray_structure,
x2 = self.model.get_xray_structure())
gamma_local = grid_pair[0]
weight_local = grid_pair[1]
self.model.restraints_manager.geometry.\
den_manager.gamma = gamma_local
self.model.restraints_manager.geometry.\
den_manager.weight = weight_local
cycle = 0
self.model.restraints_manager.geometry.\
den_manager.current_cycle = cycle+1
num_den_cycles = self.model.restraints_manager.geometry.\
den_manager.num_cycles
if self.params.den.optimize and \
self.nproc != Auto and \
self.nproc > 1:
local_log = sys.stdout
elif self.params.den.optimize and \
self.nproc == 1:
if self.verbose:
local_log = self.log
else:
local_log = StringIO()
else:
local_log = self.log
print >> self.log, " ...trying gamma %.1f, weight %.1f" % (
gamma_local, weight_local)
while cycle < num_den_cycles:
#if self.model.restraints_manager.geometry.\
# generic_restraints_manager.den_manager.current_cycle == \
# self.model.restraints_manager.geometry.\
# generic_restraints_manager.den_manager.torsion_mid_point+1:
# self.params.tardy.\
# prolsq_repulsion_function_changes.k_rep = 1.0
print >> local_log, "DEN cycle %d" % (cycle+1)
#print >> local_log, "Random seed: %d" % flex.get_random_seed()
r_free = self.fmodels.fmodel_xray().r_free()
print >> local_log, "rfree at start of SA cycle: %.4f" % r_free
print >> local_log, "k_rep = %.2f" % \
self.params.tardy.\
prolsq_repulsion_function_changes.k_rep
tardy.run(
fmodels=self.fmodels,
model=self.model,
target_weights=self.target_weights,
params=self.params.tardy,
log=local_log,
format_for_phenix_refine=True,
call_back_after_step=False)
if self.params.den.bulk_solvent_and_scale:
self.bulk_solvent_and_scale(log=local_log)
self.fmodels.fmodel_xray().xray_structure = self.model.get_xray_structure()
if self.params.den.refine_adp:
self.adp_refinement(log=local_log)
self.model.torsion_ncs_restraints_update(log=local_log)
cycle += 1
self.model.restraints_manager.geometry.\
den_manager.current_cycle += 1
r_free = self.fmodels.fmodel_xray().r_free()
print >> local_log, "rfree at end of SA cycle: %f" % r_free
r_free = self.fmodels.fmodel_xray().r_free()
step_xray_structure = self.fmodels.fmodel_xray().\
xray_structure.deep_copy_scatterers().scatterers()
step_eq_distances = self.model.restraints_manager.geometry.\
den_manager.get_current_eq_distances()
return (gamma_local,
weight_local,
r_free,
step_xray_structure,
step_eq_distances)
def exercise_restrained_refinement(options):
import random
random.seed(1)
flex.set_random_seed(1)
xs0 = smtbx.development.random_xray_structure(
sgtbx.space_group_info('P1'),
n_scatterers=options.n_scatterers,
elements="random")
for sc in xs0.scatterers():
sc.flags.set_grad_site(True)
sc0 = xs0.scatterers()
uc = xs0.unit_cell()
mi = xs0.build_miller_set(anomalous_flag=False, d_min=options.resolution)
fo_sq = mi.structure_factors_from_scatterers(
xs0, algorithm="direct").f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))
i, j, k, l = random.sample(xrange(options.n_scatterers), 4)
bond_proxies = geometry_restraints.shared_bond_simple_proxy()
w = 1e9
d_ij = uc.distance(sc0[i].site, sc0[j].site)*0.8
bond_proxies.append(geom.bond_simple_proxy(
i_seqs=(i, j),
distance_ideal=d_ij,
weight=w))
d_jk = uc.distance(sc0[j].site, sc0[k].site)*0.85
bond_proxies.append(geom.bond_simple_proxy(
i_seqs=(j, k),
distance_ideal=d_jk,
weight=w))
d_ki = min(uc.distance(sc0[k].site, sc0[i].site)*0.9, (d_ij + d_jk)*0.8)
bond_proxies.append(geom.bond_simple_proxy(
i_seqs=(k, i),
distance_ideal=d_ki,
weight=w))
d_jl = uc.distance(sc0[j].site, sc0[l].site)*0.9
bond_proxies.append(geom.bond_simple_proxy(
i_seqs=(j, l),
distance_ideal=d_jl,
weight=w))
d_lk = min(uc.distance(sc0[l].site, sc0[k].site)*0.8, 0.75*(d_jk + d_jl))
bond_proxies.append(geom.bond_simple_proxy(
i_seqs=(l, k),
distance_ideal=d_jl,
weight=w))
restraints_manager = restraints.manager(bond_proxies=bond_proxies)
xs1 = xs0.deep_copy_scatterers()
xs1.shake_sites_in_place(rms_difference=0.1)
def ls_problem():
xs = xs1.deep_copy_scatterers()
reparametrisation = constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=smtbx.utils.connectivity_table(xs),
temperature=20)
return least_squares.crystallographic_ls(
fo_sq.as_xray_observations(),
reparametrisation=reparametrisation,
restraints_manager=restraints_manager)
gradient_threshold, step_threshold = 1e-6, 1e-6
eps = 5e-3
ls = ls_problem()
t = wall_clock_time()
cycles = normal_eqns_solving.naive_iterations(
ls,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold,
track_all=True)
if options.verbose:
print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed())
sc = ls.xray_structure.scatterers()
for p in bond_proxies:
d = uc.distance(*[ sc[i_pair].site for i_pair in p.i_seqs ])
assert approx_equal(d, p.distance_ideal, eps)
ls = ls_problem()
t = wall_clock_time()
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold,
tau=1e-3,
track_all=True)
if options.verbose:
print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed())
sc = ls.xray_structure.scatterers()
sc = ls.xray_structure.scatterers()
for p in bond_proxies:
d = uc.distance(*[ sc[i].site for i in p.i_seqs ])
assert approx_equal(d, p.distance_ideal, eps)
def exercise_hbond_as_cif_loop():
xs = sucrose()
for sc in xs.scatterers():
sc.flags.set_grad_site(True)
radii = [
covalent_radii.table(elt).radius() for elt in
xs.scattering_type_registry().type_index_pairs_as_dict() ]
asu_mappings = xs.asu_mappings(
buffer_thickness=2*max(radii) + 0.5)
pair_asu_table = crystal.pair_asu_table(asu_mappings)
pair_asu_table.add_covalent_pairs(
xs.scattering_types(),
tolerance=0.5)
hbonds = [
geometry.hbond(1,5, sgtbx.rt_mx('-X,0.5+Y,2-Z')),
geometry.hbond(5,14, sgtbx.rt_mx('-X,-0.5+Y,1-Z')),
geometry.hbond(7,10, sgtbx.rt_mx('1+X,+Y,+Z')),
geometry.hbond(10,0),
geometry.hbond(12,14, sgtbx.rt_mx('-1-X,0.5+Y,1-Z')),
geometry.hbond(14,12, sgtbx.rt_mx('-1-X,-0.5+Y,1-Z')),
geometry.hbond(16,7)
]
loop = geometry.hbonds_as_cif_loop(
hbonds, pair_asu_table, xs.scatterers().extract_labels(),
sites_frac=xs.sites_frac()).loop
s = StringIO()
print >> s, loop
assert not show_diff(s.getvalue(), """\
loop_
_geom_hbond_atom_site_label_D
_geom_hbond_atom_site_label_H
_geom_hbond_atom_site_label_A
_geom_hbond_distance_DH
_geom_hbond_distance_HA
_geom_hbond_distance_DA
_geom_hbond_angle_DHA
_geom_hbond_site_symmetry_A
O2 H2 O4 0.8200 2.0636 2.8635 165.0 2_557
O4 H4 O9 0.8200 2.0559 2.8736 174.9 2_546
O5 H5 O7 0.8200 2.0496 2.8589 169.0 1_655
O7 H7 O1 0.8200 2.0573 2.8617 166.8 .
O8 H8 O9 0.8200 2.1407 2.8943 152.8 2_456
O9 H9 O8 0.8200 2.1031 2.8943 162.1 2_446
O10 H10 O5 0.8200 2.0167 2.7979 159.1 .
""")
# with a covariance matrix
flex.set_random_seed(1)
vcv_matrix = matrix.diag(
flex.random_double(size=xs.n_parameters(), factor=1e-5))\
.as_flex_double_matrix().matrix_symmetric_as_packed_u()
loop = geometry.hbonds_as_cif_loop(
hbonds, pair_asu_table, xs.scatterers().extract_labels(),
sites_frac=xs.sites_frac(),
covariance_matrix=vcv_matrix,
parameter_map=xs.parameter_map()).loop
s = StringIO()
print >> s, loop
assert not show_diff(s.getvalue(), """\
loop_
_geom_hbond_atom_site_label_D
_geom_hbond_atom_site_label_H
_geom_hbond_atom_site_label_A
_geom_hbond_distance_DH
_geom_hbond_distance_HA
_geom_hbond_distance_DA
_geom_hbond_angle_DHA
_geom_hbond_site_symmetry_A
O2 H2 O4 0.82(3) 2.06(3) 2.86(3) 165.0(18) 2_557
O4 H4 O9 0.82(4) 2.06(4) 2.87(4) 175(2) 2_546
O5 H5 O7 0.82(2) 2.05(2) 2.859(19) 169.0(18) 1_655
O7 H7 O1 0.82(2) 2.06(2) 2.86(2) 167(2) .
O8 H8 O9 0.82(3) 2.14(3) 2.89(3) 153(3) 2_456
O9 H9 O8 0.82(3) 2.10(3) 2.89(3) 162(2) 2_446
O10 H10 O5 0.82(3) 2.02(3) 2.80(3) 159(3) .
""")
cell_vcv = flex.pow2(matrix.diag(flex.random_double(size=6,factor=1e-1))\
.as_flex_double_matrix().matrix_symmetric_as_packed_u())
loop = geometry.hbonds_as_cif_loop(
hbonds, pair_asu_table, xs.scatterers().extract_labels(),
sites_frac=xs.sites_frac(),
covariance_matrix=vcv_matrix,
cell_covariance_matrix=cell_vcv,
parameter_map=xs.parameter_map()).loop
s = StringIO()
print >> s, loop
assert not show_diff(s.getvalue(), """\
loop_
_geom_hbond_atom_site_label_D
_geom_hbond_atom_site_label_H
_geom_hbond_atom_site_label_A
_geom_hbond_distance_DH
_geom_hbond_distance_HA
_geom_hbond_distance_DA
_geom_hbond_angle_DHA
_geom_hbond_site_symmetry_A
O2 H2 O4 0.82(3) 2.06(4) 2.86(4) 165.0(18) 2_557
O4 H4 O9 0.82(4) 2.06(4) 2.87(4) 175(2) 2_546
O5 H5 O7 0.82(2) 2.05(2) 2.86(2) 169.0(18) 1_655
O7 H7 O1 0.82(2) 2.06(3) 2.86(3) 167(2) .
O8 H8 O9 0.82(3) 2.14(4) 2.89(4) 153(3) 2_456
O9 H9 O8 0.82(3) 2.10(3) 2.89(4) 162(2) 2_446
O10 H10 O5 0.82(3) 2.02(3) 2.80(3) 159(3) .
""")
def exercise(space_group_info,
fixed_random_seed=True,
shifted_origin=None,
elements=None,
d_min=0.8,
grid_resolution_factor=1/3.,
verbose=False,
**kwds):
if elements is None:
n_C = 5
n_O = 1
n_N = 1
elements = ["C"]*n_C + ["O"]*n_O + ["N"]*n_N
if verbose:
print elements
target_space_group_type = space_group_info.type()
hall = sgtbx.space_group_symbols(
target_space_group_type.lookup_symbol()).hall()
print hall
if fixed_random_seed:
random.seed(1)
flex.set_random_seed(1)
# Generate a random structure in real space,
# compute its structure factors,
# that we will try to recover the symmetry of
target_structure = random_structure.xray_structure(
space_group_info=space_group_info,
elements=elements,
use_u_iso=True,
random_u_iso=True,
random_u_iso_scale=0.04,
use_u_aniso=False,
)
if shifted_origin is None:
shifted_origin = flex.random_double(3)
shifted_origin = mat.col(shifted_origin)
if verbose:
print "new origin = (%.3f, %.3f, %.3f)" % shifted_origin.elems
print
target_structure_in_p1 = target_structure\
.expand_to_p1().apply_shift(shifted_origin)
target_f_in_p1 = miller.build_set(
crystal_symmetry=target_structure_in_p1,
anomalous_flag=False,
d_min=d_min
).structure_factors_from_scatterers(
xray_structure=target_structure_in_p1,
algorithm="direct").f_calc()
# Recover space group?
sf_symm = symmetry_search.structure_factor_symmetry(target_f_in_p1)
if verbose:
print sf_symm
solution_hall, target_hall = [
sgi.as_reference_setting().type().hall_symbol()
for sgi in (sf_symm.space_group_info,
target_structure.space_group_info()) ]
assert solution_hall == target_hall, (solution_hall, target_hall)
# Shift maximises goodness of symmetry?
gos, solution_f = sf_symm.symmetrised_structure_factors()
if space_group_info.type().hall_symbol() != ' P 1':
assert gos.correlation > 0.99
assert gos.gradient == (0, 0, 0)
gos_away_from_max = sf_symm.symmetrised_structure_factors(
delta=mat.col((0.1, 0.1, 0.1)))[0]
assert gos_away_from_max.correlation < 0.9, gos_away_from_max.correlation
# Recovered origin
"""The sequence of transform read:
----->target structure
^ V
^ V (1, shifted_origin=sigma)
^ V
^ shifted target
^ V
^ V sf_symm.cb_op_to_primitive = (P, 0)
^ V
^ shifted target in primitive cell = shifted solution in primitive cell
^ V
^ V (1, -sf_symm.origin=-s)
^ V
^ solution in primitive cell
^ V
^ V solution_to_target_cb_op = (Q, q)
^ V
^------------
The total transfrom from the target structure back to it reads
(QP, q') with q' = (Q,q)(-s + P sigma) = (Q,q)delta_o
with
delta_o = sf_symm.cb_op_to_primitive(shifted_origin) - sf_symm.origin
(Q, q') must leave the target structure space group invariant.
Most of the time Q is the unit matrix and the test boils down to check
whether delta is an allowed origin shift after changing to the input cell
but it does not hurt to do the more general test all the time.
"""
solution_to_target_cb_op = (
target_structure.space_group_info()
.change_of_basis_op_to_reference_setting().inverse()
*
sf_symm.space_group_info
.change_of_basis_op_to_reference_setting())
if verbose:
print
print "solution -> target: %s" % solution_to_target_cb_op.as_xyz()
delta_o = (mat.col(sf_symm.cb_op_to_primitive(shifted_origin))
- sf_symm.origin)
delta = mat.col(solution_to_target_cb_op(delta_o))
stabilising_cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(
(solution_to_target_cb_op*sf_symm.cb_op_to_primitive).c().r(),
sgtbx.tr_vec((delta*72).as_int()*2, tr_den=144)))
# guarding against rounding errors on some platforms (e.g. FC8)
# meaning that (delta*144).as_int() would not work.
target_sg = target_structure.space_group()
assert target_sg == target_sg.change_basis(stabilising_cb_op)
def try_den_weight_torsion(self, grid_pair):
#backup_k_rep = self.params.tardy.\
# prolsq_repulsion_function_changes.k_rep
local_seed = int(self.random_seed+grid_pair[1])
flex.set_random_seed(value=local_seed)
random.seed(local_seed)
self.fmodels.fmodel_xray().xray_structure.replace_scatterers(
self.save_scatterers_local.deep_copy())
self.fmodels.update_xray_structure(
xray_structure = self.fmodels.fmodel_xray().xray_structure,
update_f_calc = True)
utils.assert_xray_structures_equal(
x1 = self.fmodels.fmodel_xray().xray_structure,
x2 = self.model.xray_structure)
gamma_local = grid_pair[0]
weight_local = grid_pair[1]
self.model.restraints_manager.geometry.\
den_manager.gamma = gamma_local
self.model.restraints_manager.geometry.\
den_manager.weight = weight_local
cycle = 0
self.model.restraints_manager.geometry.\
den_manager.current_cycle = cycle+1
num_den_cycles = self.model.restraints_manager.geometry.\
den_manager.num_cycles
if self.params.den.optimize and \
self.nproc != Auto and \
self.nproc > 1:
local_log = sys.stdout
elif self.params.den.optimize and \
self.nproc == 1:
if self.verbose:
local_log = self.log
else:
local_log = StringIO()
else:
local_log = self.log
print >> self.log, " ...trying gamma %.1f, weight %.1f" % (
gamma_local, weight_local)
while cycle < num_den_cycles:
#if self.model.restraints_manager.geometry.\
# generic_restraints_manager.den_manager.current_cycle == \
# self.model.restraints_manager.geometry.\
# generic_restraints_manager.den_manager.torsion_mid_point+1:
# self.params.tardy.\
# prolsq_repulsion_function_changes.k_rep = 1.0
print >> local_log, "DEN cycle %d" % (cycle+1)
#print >> local_log, "Random seed: %d" % flex.get_random_seed()
r_free = self.fmodels.fmodel_xray().r_free()
print >> local_log, "rfree at start of SA cycle: %.4f" % r_free
print >> local_log, "k_rep = %.2f" % \
self.params.tardy.\
prolsq_repulsion_function_changes.k_rep
tardy.run(
fmodels=self.fmodels,
model=self.model,
target_weights=self.target_weights,
params=self.params.tardy,
log=local_log,
format_for_phenix_refine=True,
call_back_after_step=False)
if self.params.den.bulk_solvent_and_scale:
self.bulk_solvent_and_scale(log=local_log)
self.fmodels.fmodel_xray().xray_structure = self.model.xray_structure
if self.params.den.refine_adp:
self.adp_refinement(log=local_log)
self.model.restraints_manager.geometry.update_dihedral_ncs_restraints(
sites_cart=self.model.xray_structure.sites_cart(),
pdb_hierarchy=self.model.pdb_hierarchy(sync_with_xray_structure=True),
log=local_log)
cycle += 1
self.model.restraints_manager.geometry.\
den_manager.current_cycle += 1
r_free = self.fmodels.fmodel_xray().r_free()
print >> local_log, "rfree at end of SA cycle: %f" % r_free
r_free = self.fmodels.fmodel_xray().r_free()
step_xray_structure = self.fmodels.fmodel_xray().\
xray_structure.deep_copy_scatterers().scatterers()
step_eq_distances = self.model.restraints_manager.geometry.\
den_manager.get_current_eq_distances()
return (gamma_local,
weight_local,
r_free,
step_xray_structure,
step_eq_distances)
from __future__ import division
from cctbx.array_family import flex
import mmtbx.f_model
from cctbx.development import random_structure
from cctbx.development import debug_utils
from cctbx import sgtbx
import random
import sys
from mmtbx.scaling import outlier_rejection
from cctbx.xray import observation_types
from cctbx.development import debug_utils
if 1:
random.seed(0)
flex.set_random_seed(value=0)
def exercise(
d_min=3.5,
k_sol=0.3,
b_sol=60.0,
b_cart=[0, 0, 0, 0, 0, 0],
anomalous_flag=False,
scattering_table="it1992",
space_group_info=None,
):
space_groups = [str(space_group_info)]
for sg in space_groups:
### get random structure
xray_structure = random_structure.xray_structure(
space_group_info=sgtbx.space_group_info(sg),
def __init__(self, model, params, log = None):
self.log = log
self.params = params
self.model = model
self._neutralize_scatterers()
if not model.crystal_symmetry() or not model.crystal_symmetry().unit_cell():
# Make it up
from cctbx.maptbx.box import shift_and_box_model
model = shift_and_box_model(model, shift_model=False)
self.pdb_hierarchy = model.get_hierarchy()
self.crystal_symmetry = model.crystal_symmetry()
if(self.log is None): self.log = sys.stdout
self.xray_structure = model.get_xray_structure()
asc = self.pdb_hierarchy.atom_selection_cache(
special_position_settings=crystal.special_position_settings(
crystal_symmetry = self.crystal_symmetry))
if(self.params.random_seed is not None):
random.seed(self.params.random_seed)
flex.set_random_seed(self.params.random_seed)
self.top_selection = flex.smart_selection(
flags=flex.bool(self.xray_structure.scatterers().size(), True))
if(self.params.selection is not None):
self.top_selection = flex.smart_selection(
flags=asc.selection(self.params.selection))
self._rotate_about_axis()
self._process_adp()
self._process_sites()
self._process_occupancies()
self._put_in_box()
self._change_of_basis()
# Up to this point we are done with self.xray_structure
self.model.set_xray_structure(self.xray_structure)
self.pdb_hierarchy = self.model.get_hierarchy()
# Now only manipulations that use self.pdb_hierarchy are done
### segID manipulations
if (params.set_seg_id_to_chain_id):
if (params.clear_seg_id):
raise Sorry("Parameter conflict - set_seg_id_to_chain_id=True and "+
"clear_seg_id=True. Please choose only one of these options.")
for atom in self.pdb_hierarchy.atoms():
labels = atom.fetch_labels()
atom.segid = "%-4s" % labels.chain_id
elif (params.clear_seg_id):
for atom in self.pdb_hierarchy.atoms():
atom.segid = " "
if(self.params.set_chemical_element_simple_if_necessary or
self.params.rename_chain_id.old_id or
self.params.renumber_residues or self.params.increment_resseq or
self.params.convert_semet_to_met or
self.params.convert_met_to_semet or
self.params.set_charge.charge or
self.params.truncate_to_polyala or
self.params.truncate_to_polygly or
self.params.remove_alt_confs or
self.params.move_waters_last or
self.params.remove_fraction or
self.params.keep or
self.params.remove):
# del self.xray_structure # it is invalide below this point
self._set_chemical_element_simple_if_necessary()
self._rename_chain_id()
self._renumber_residues()
self._convert_semet_to_met()
self._convert_met_to_semet()
self._set_atomic_charge()
self._truncate_to_poly_ala()
self._truncate_to_poly_gly()
self._remove_alt_confs()
self._move_waters()
self._remove_atoms()
self._apply_keep_remove()
# Here goes really nasty hack. Never repeat it.
# It is here because I don't have clear idea about how to handle
# such dramatic changes in number of atoms etc that just was performed
# for hierarchy.
self.pdb_hierarchy.reset_atom_i_seqs()
self.pdb_hierarchy.atoms_reset_serial()
self.model._pdb_hierarchy = self.pdb_hierarchy
self.model._xray_structure = self.pdb_hierarchy.extract_xray_structure(
crystal_symmetry=self.model.crystal_symmetry())
self.model._update_atom_selection_cache()
self.model._update_has_hd()
self.model.get_hierarchy().atoms().reset_i_seq()
def exercise(d_min=5, random_seed=1111111):
inp = get_pdb_inputs(pdb_str=pdb_str)
xrs_good = inp.xrs.deep_copy_scatterers()
target_map = get_tmo(inp=inp, d_min=d_min)
inp.ph.write_pdb_file(file_name="start.pdb")
show(prefix="GOOD",
pdb_hierarchy=inp.ph,
tm=target_map,
xrs=xrs_good,
grm=inp.grm.geometry)
#
sites_cart_reference = []
selections_reference = []
pdb_hierarchy_reference = inp.ph.deep_copy()
pdb_hierarchy_reference.reset_i_seq_if_necessary()
for model in inp.ph.models():
for chain in model.chains():
for residue in chain.residues():
sites_cart_reference.append(residue.atoms().extract_xyz())
selections_reference.append(residue.atoms().extract_i_seq())
#
sites_cart_reference_for_chi_only = []
selections_reference_for_chi_only = []
for model in inp.ph.models():
for chain in model.chains():
for residue in chain.residues():
s1 = flex.vec3_double()
s2 = flex.size_t()
for atom in residue.atoms():
if (not atom.name.strip().upper() in ["O"]):
s1.append(atom.xyz)
s2.append(atom.i_seq)
sites_cart_reference_for_chi_only.append(s1)
selections_reference_for_chi_only.append(s2)
#
xrs_poor = shake_sites(xrs=xrs_good.deep_copy_scatterers(),
random=False,
shift=2.0,
grm=inp.grm)
inp.ph.adopt_xray_structure(xrs_poor)
inp.ph.write_pdb_file(file_name="poor.pdb")
#
for use_reference_torsion in [
"no", "yes_add_once", "yes_add_per_residue", "yes_manual"
]:
es = inp.grm.energies_sites(
sites_cart=xrs_good.sites_cart()) # it's essential to update grm
inp.ph.adopt_xray_structure(xrs_poor)
random.seed(random_seed)
flex.set_random_seed(random_seed)
print "*" * 79
print "use_reference_torsion:", use_reference_torsion
print "*" * 79
show(prefix="START",
pdb_hierarchy=inp.ph,
tm=target_map,
xrs=xrs_poor,
grm=inp.grm.geometry)
#
if (use_reference_torsion == "yes_add_per_residue"):
inp.grm.geometry.remove_chi_torsion_restraints_in_place()
for sites_cart, selection in zip(sites_cart_reference,
selections_reference):
inp.grm.geometry.add_chi_torsion_restraints_in_place(
pdb_hierarchy=pdb_hierarchy_reference,
sites_cart=sites_cart,
selection=selection,
chi_angles_only=True,
sigma=1)
if (use_reference_torsion == "yes_add_once"):
inp.grm.geometry.remove_chi_torsion_restraints_in_place()
inp.grm.geometry.add_chi_torsion_restraints_in_place(
pdb_hierarchy=pdb_hierarchy_reference,
sites_cart=xrs_good.sites_cart(),
chi_angles_only=True,
sigma=1)
if (use_reference_torsion == "yes_manual"):
inp.grm.geometry.remove_chi_torsion_restraints_in_place()
for sites_cart, selection in zip(
sites_cart_reference_for_chi_only,
selections_reference_for_chi_only):
inp.grm.geometry.add_chi_torsion_restraints_in_place(
pdb_hierarchy=pdb_hierarchy_reference,
sites_cart=sites_cart,
selection=selection,
chi_angles_only=True,
sigma=1)
#
tmp = xrs_poor.deep_copy_scatterers()
rsr_simple_refiner = individual_sites.simple(
target_map=target_map.data,
selection=flex.bool(tmp.scatterers().size(), True),
real_space_gradients_delta=d_min / 4,
max_iterations=500,
geometry_restraints_manager=inp.grm.geometry)
refined = individual_sites.refinery(refiner=rsr_simple_refiner,
optimize_weight=True,
xray_structure=tmp,
start_trial_weight_value=50,
rms_bonds_limit=0.02,
rms_angles_limit=2.0)
assert refined.sites_cart_result is not None
tmp = tmp.replace_sites_cart(refined.sites_cart_result)
inp.ph.adopt_xray_structure(tmp)
show(prefix="FINAL",
pdb_hierarchy=inp.ph,
tm=target_map,
xrs=tmp,
grm=inp.grm.geometry)
inp.ph.write_pdb_file(file_name="final_%s.pdb" %
str(use_reference_torsion))
from cctbx import xray
import mmtbx.utils
from scitbx.array_family import flex
import random
# For lbfgs class
from cctbx import xray
from cctbx import crystal
from cctbx.array_family import flex
import scitbx.lbfgs
from libtbx import adopt_init_args
from stdlib import math
if (1): # fixed random seed
random.seed(1)
flex.set_random_seed(1)
class lbfgs(object):
def __init__(self,
xray_structure,
geometry_restraints,
states,
max_iterations=100,
min_iterations=0,
verbose=0,
correct_special_position_tolerance=1.0):
adopt_init_args(self, locals())
self.f = None
self.correct_special_position_tolerance = correct_special_position_tolerance
self.x = flex.double(self.xray_structure.n_parameters(), 0)
def run_test(params, pdb_files, other_files, callback=None, log=None):
if (log is None): log = sys.stdout
if (params.random_seed is not None):
random.seed(params.random_seed)
flex.set_random_seed(value=params.random_seed)
#
if (len(pdb_files) != 0):
print >> log, "PDB files:"
for file_name in pdb_files:
print >> log, " ", file_name
print >> log
if (len(other_files) != 0):
print >> log, "Other files:"
for file_name in other_files:
print >> log, " ", file_name
print >> log
#
assert len(pdb_files) in [1, 2]
#
pdb_interpretation_params = pdb_interpretation.master_params.extract()
pdb_interpretation_params.dihedral_function_type \
= params.dihedral_function_type
processed_pdb_files = pdb_interpretation.run(
args=pdb_files[-1:]+other_files,
params=pdb_interpretation_params,
strict_conflict_handling=False,
substitute_non_crystallographic_unit_cell_if_necessary=True,
return_all_processed_pdb_files=True,
log=log)
assert len(processed_pdb_files) == 1
print >> log
#
xs = processed_pdb_files[0].xray_structure()
geo_manager = processed_pdb_files[0].geometry_restraints_manager()
labels = [sc.label for sc in xs.scatterers()]
ideal_sites_cart = xs.sites_cart()
sites = ideal_sites_cart
masses = xs.atomic_weights()
tardy_tree_simple_connectivity = geo_manager.construct_tardy_tree(sites=sites)
rmsd_calculator = tardy_tree_simple_connectivity.rmsd_calculator()
#
if (params.tardy_displacements is not None):
def get_tardy_model_no_potential():
return scitbx.rigid_body.tardy_model(
labels=labels,
sites=sites,
masses=masses,
tardy_tree=tardy_tree_simple_connectivity,
potential_obj=None)
def get_tardy_model_no_density():
tardy_tree = scitbx.graph.tardy_tree.construct(
n_vertices=len(sites), edge_list=[])
tardy_tree.build_tree()
potential_obj = potential_object(
density_map=None,
geo_manager=geo_manager,
reduced_geo_manager=None,
prolsq_repulsion_function_changes=
params.prolsq_repulsion_function_changes,
real_space_gradients_delta=None,
real_space_target_weight=None,
ideal_sites_cart=None)
return scitbx.rigid_body.tardy_model(
labels=labels,
sites=sites,
masses=masses,
tardy_tree=tardy_tree,
potential_obj=potential_obj)
if (params.tardy_displacements is Auto):
auto_params = params.tardy_displacements_auto
target_rmsd = \
params.structure_factors_high_resolution \
* auto_params.rmsd_vs_high_resolution_factor
target_rmsd_tol = \
params.structure_factors_high_resolution \
* auto_params.rmsd_tolerance
assert target_rmsd > 0
assert target_rmsd_tol > 0
print >> log, "Random displacements (%s):" \
% params.tardy_displacements_auto.parameterization
print >> log, " high resolution: %.6g" \
% params.structure_factors_high_resolution
print >> log, " target rmsd: %.6g" % target_rmsd
print >> log, " target rmsd tolerance: %.6g" % target_rmsd_tol
log.flush()
def raise_max_steps_exceeded(var_name, rmsd_history):
msg = [
"tardy_displacements_auto.max_steps exceeded:",
" % -13s rmsd" % var_name]
for var_rmsd in rmsd_history:
msg.append(" %13.6e %13.6e" % var_rmsd)
raise Sorry("\n".join(msg))
if (params.tardy_displacements_auto.parameterization == "cartesian"):
multiplier = 1.5
rmsd_history = []
for i_step in xrange(auto_params.max_steps):
sites = cartesian_random_displacements(
sites_cart=ideal_sites_cart,
target_rmsd=target_rmsd*multiplier)
tardy_model = get_tardy_model_no_density()
tardy_model.minimization(max_iterations=20)
sites = tardy_model.sites_moved()
sites_moved = sites
rmsd = rmsd_calculator(sites_moved, ideal_sites_cart)
rmsd_history.append((multiplier, rmsd))
print >> log, " multiplier, rmsd: %13.6e, %13.6e" \
% rmsd_history[-1]
log.flush()
if (rmsd < target_rmsd - target_rmsd_tol):
if (rmsd != 0):
multiplier = min(
multiplier*2, max(
multiplier*1.2,
target_rmsd / rmsd))
else:
multiplier *= 2
else:
if (rmsd <= target_rmsd + target_rmsd_tol):
tardy_model.minimization(max_iterations=500)
sites = tardy_model.sites_moved()
sites_moved = sites
rmsd = rmsd_calculator(sites_moved, ideal_sites_cart)
rmsd_history.append((0, rmsd))
print >> log, " multiplier, rmsd: %13.6e, %13.6e" \
% rmsd_history[-1]
log.flush()
break
multiplier *= max(0.5, target_rmsd/rmsd)
else:
raise_max_steps_exceeded(
var_name="multiplier", rmsd_history=rmsd_history)
del rmsd_history
print >> log, " actual rmsd: %.6g" % rmsd
print >> log
elif (params.tardy_displacements_auto.parameterization == "constrained"):
tardy_model = get_tardy_model_no_potential()
tardy_model.assign_random_velocities()
delta_t = auto_params.first_delta_t
rmsd_history = []
assert auto_params.max_steps > 0
for i_step in xrange(auto_params.max_steps):
prev_q = tardy_model.pack_q()
prev_qd = tardy_model.pack_qd()
tardy_model.dynamics_step(delta_t=delta_t)
sites_moved = tardy_model.sites_moved()
rmsd = rmsd_calculator(sites_moved, ideal_sites_cart)
rmsd_history.append((delta_t, rmsd))
if (rmsd < target_rmsd - target_rmsd_tol):
delta_t *= 2 - rmsd / target_rmsd
else:
if (rmsd <= target_rmsd + target_rmsd_tol):
break
tardy_model.unpack_q(q_packed=prev_q)
tardy_model.unpack_qd(qd_packed=prev_qd)
delta_t *= 0.5
prev_q = None
prev_qd = None
else:
raise_max_steps_exceeded(
var_name="delta_t", rmsd_history=rmsd_history)
del rmsd_history
print >> log, " actual rmsd: %.6g" % rmsd
print >> log, " tardy_displacements=%s" % ",".join(
["%.6g" % v for v in tardy_model.pack_q()])
print >> log
sites = tardy_model.sites_moved()
else:
raise AssertionError
else:
tardy_model = get_tardy_model_no_potential()
q = tardy_model.pack_q()
if (len(params.tardy_displacements) != len(q)):
print >> log, "tardy_displacements:", params.tardy_displacements
hinge_edges = tardy_model.tardy_tree.cluster_manager.hinge_edges
assert len(hinge_edges) == tardy_model.bodies_size()
dofej = tardy_model.degrees_of_freedom_each_joint()
qsej = tardy_model.q_size_each_joint()
for ib,(i,j) in enumerate(hinge_edges):
if (i == -1): si = "root"
else: si = tardy_model.labels[i]
sj = tardy_model.labels[j]
print >> log, "%21s - %-21s: %d dof, %d q_size" % (
si, sj, dofej[ib], qsej[ib])
print >> log, "Zero displacements:"
print >> log, " tardy_displacements=%s" % ",".join(
[str(v) for v in q])
raise Sorry("Incompatible tardy_displacements.")
tardy_model.unpack_q(q_packed=flex.double(params.tardy_displacements))
sites = tardy_model.sites_moved()
#
if (params.emulate_cartesian):
tardy_tree = scitbx.graph.tardy_tree.construct(
n_vertices=len(sites), edge_list=[])
tardy_tree.build_tree()
else:
tardy_tree = tardy_tree_simple_connectivity
print >> log, "tardy_tree summary:"
tardy_tree.show_summary(vertex_labels=labels, out=log, prefix=" ")
print >> log
#
if (len(pdb_files) == 2):
ideal_pdb_inp = iotbx.pdb.input(file_name=pdb_files[0])
ideal_pdb_hierarchy = ideal_pdb_inp.construct_hierarchy()
assert ideal_pdb_hierarchy.is_similar_hierarchy(
processed_pdb_files[0].all_chain_proxies.pdb_hierarchy)
ideal_sites_cart = ideal_pdb_hierarchy.atoms().extract_xyz()
xs.set_sites_cart(sites_cart=ideal_sites_cart)
fft_map = xs.structure_factors(
d_min=params.structure_factors_high_resolution).f_calc().fft_map()
fft_map.apply_sigma_scaling()
#
assert not params.orca_experiments or not params.emulate_cartesian
if (params.orca_experiments):
from mmtbx.refinement import orca
x = orca.expand(
labels=labels,
sites_cart=sites,
masses=masses,
geo_manager=geo_manager)
labels = x.labels
sites = x.sites_cart
masses = x.masses
geo_manager = x.geo_manager
tardy_tree = x.tardy_tree
x_ideal_sites_cart = flex.vec3_double()
for i_orc,i_seq in x.indices:
x_ideal_sites_cart.append(ideal_sites_cart[i_seq])
ideal_sites_cart = x_ideal_sites_cart
rmsd_calculator = x.rmsd_calculator(
tardy_tree_rmsd_calculator=rmsd_calculator)
#
if (params.emulate_cartesian or params.keep_all_restraints):
reduced_geo_manager = None
else:
reduced_geo_manager = geo_manager.reduce_for_tardy(tardy_tree=tardy_tree)
real_space_gradients_delta = \
params.structure_factors_high_resolution \
* params.real_space_gradients_delta_resolution_factor
potential_obj = potential_object(
density_map=fft_map.real_map(),
geo_manager=geo_manager,
reduced_geo_manager=reduced_geo_manager,
prolsq_repulsion_function_changes=params.prolsq_repulsion_function_changes,
real_space_gradients_delta=real_space_gradients_delta,
real_space_target_weight=params.real_space_target_weight,
ideal_sites_cart=ideal_sites_cart,
site_labels=labels,
orca_experiments=params.orca_experiments)
tardy_model = scitbx.rigid_body.tardy_model(
labels=labels,
sites=sites,
masses=masses,
tardy_tree=tardy_tree,
potential_obj=potential_obj)
mmtbx.refinement.tardy.action(
tardy_model=tardy_model,
params=params,
rmsd_calculator=rmsd_calculator,
callback=callback,
log=log)
print >> log
import cctbx.geometry_restraints.flags
import cctbx.geometry_restraints.manager
import cctbx.geometry_restraints.lbfgs
from cctbx import xray
from cctbx import crystal
import cctbx.crystal.coordination_sequences
from cctbx import sgtbx
from cctbx.array_family import flex
from scitbx import matrix as mx
import scitbx.lbfgs
from libtbx.str_utils import format_value
from itertools import count
import sys
if (1):
flex.set_random_seed(0)
class restraint_parameters(object):
def __init__(self, distance_ideal, weight):
self.distance_ideal = distance_ideal
self.weight = weight
restraint_parameters_si_o = restraint_parameters(1.61, 2.0)
restraint_parameters_o_si_o = restraint_parameters(2.629099, 0.41)
restraint_parameters_si_o_si = restraint_parameters(3.070969, 0.2308)
def setup_bond_params_table(structure, bond_sym_table):
scatterers = structure.scatterers()
t = geometry_restraints.bond_params_table(scatterers.size())
for i_seq,bond_sym_dict in enumerate(bond_sym_table):
def exercise(d_min=5, random_seed=1111111):
inp = get_pdb_inputs(pdb_str=pdb_str)
xrs_good = inp.xrs.deep_copy_scatterers()
target_map = get_tmo(inp=inp, d_min = d_min)
inp.ph.write_pdb_file(file_name="start.pdb")
show(prefix="GOOD",
pdb_hierarchy = inp.ph,
tm=target_map,
xrs=xrs_good,
grm=inp.grm.geometry)
#
sites_cart_reference = []
selections_reference = []
pdb_hierarchy_reference = inp.ph.deep_copy()
pdb_hierarchy_reference.reset_i_seq_if_necessary()
for model in inp.ph.models():
for chain in model.chains():
for residue in chain.residues():
sites_cart_reference.append(residue.atoms().extract_xyz())
selections_reference.append(residue.atoms().extract_i_seq())
#
sites_cart_reference_for_chi_only = []
selections_reference_for_chi_only = []
for model in inp.ph.models():
for chain in model.chains():
for residue in chain.residues():
s1 = flex.vec3_double()
s2 = flex.size_t()
for atom in residue.atoms():
if(not atom.name.strip().upper() in ["O"]):
s1.append(atom.xyz)
s2.append(atom.i_seq)
sites_cart_reference_for_chi_only.append(s1)
selections_reference_for_chi_only.append(s2)
#
xrs_poor = shake_sites(xrs=xrs_good.deep_copy_scatterers(), random=False,
shift=2.0, grm=inp.grm)
inp.ph.adopt_xray_structure(xrs_poor)
inp.ph.write_pdb_file(file_name="poor.pdb")
#
for use_reference_torsion in ["no", "yes_add_once", "yes_add_per_residue",
"yes_manual"]:
es = inp.grm.energies_sites(sites_cart = xrs_good.sites_cart()) # it's essential to update grm
inp.ph.adopt_xray_structure(xrs_poor)
random.seed(random_seed)
flex.set_random_seed(random_seed)
print "*"*79
print "use_reference_torsion:", use_reference_torsion
print "*"*79
show(prefix="START",pdb_hierarchy = inp.ph, tm=target_map, xrs=xrs_poor, grm=inp.grm.geometry)
#
if(use_reference_torsion == "yes_add_per_residue"):
inp.grm.geometry.remove_chi_torsion_restraints_in_place()
for sites_cart, selection in zip(sites_cart_reference, selections_reference):
inp.grm.geometry.add_chi_torsion_restraints_in_place(
pdb_hierarchy = pdb_hierarchy_reference,
sites_cart = sites_cart,
selection = selection,
chi_angles_only = True,
sigma = 1)
if(use_reference_torsion == "yes_add_once"):
inp.grm.geometry.remove_chi_torsion_restraints_in_place()
inp.grm.geometry.add_chi_torsion_restraints_in_place(
pdb_hierarchy = pdb_hierarchy_reference,
sites_cart = xrs_good.sites_cart(),
chi_angles_only = True,
sigma = 1)
if(use_reference_torsion == "yes_manual"):
inp.grm.geometry.remove_chi_torsion_restraints_in_place()
for sites_cart, selection in zip(sites_cart_reference_for_chi_only,
selections_reference_for_chi_only):
inp.grm.geometry.add_chi_torsion_restraints_in_place(
pdb_hierarchy = pdb_hierarchy_reference,
sites_cart = sites_cart,
selection = selection,
chi_angles_only = True,
sigma = 1)
#
tmp = xrs_poor.deep_copy_scatterers()
rsr_simple_refiner = individual_sites.simple(
target_map = target_map.data,
selection = flex.bool(tmp.scatterers().size(), True),
real_space_gradients_delta = d_min/4,
max_iterations = 500,
geometry_restraints_manager = inp.grm.geometry)
refined = individual_sites.refinery(
refiner = rsr_simple_refiner,
optimize_weight = True,
xray_structure = tmp,
start_trial_weight_value = 50,
rms_bonds_limit = 0.02,
rms_angles_limit = 2.0)
assert refined.sites_cart_result is not None
tmp = tmp.replace_sites_cart(refined.sites_cart_result)
inp.ph.adopt_xray_structure(tmp)
show(prefix="FINAL",pdb_hierarchy = inp.ph, tm=target_map, xrs=tmp, grm=inp.grm.geometry)
inp.ph.write_pdb_file(file_name="final_%s.pdb"%str(use_reference_torsion))
from __future__ import division
import clipper
from cctbx import xray
from cctbx import miller
from cctbx import crystal
from cctbx.development import random_structure
from cctbx.development import debug_utils
from cctbx.array_family import flex
from libtbx.test_utils import approx_equal
import sys
if (1):
flex.set_random_seed(2)
def exercise_SFweight_spline_core(structure, d_min, verbose=0):
structure.scattering_type_registry(d_min=d_min)
f_obs = abs(structure.structure_factors(
d_min=d_min, anomalous_flag=False).f_calc())
if (0 or verbose):
f_obs.show_summary()
f_obs = miller.array(
miller_set=f_obs,
data=f_obs.data(),
sigmas=flex.sqrt(f_obs.data()))
partial_structure = xray.structure(
crystal_symmetry=structure,
scatterers=structure.scatterers()[:-2])
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure=partial_structure).f_calc()
test_set_flags = (flex.random_double(size=f_obs.indices().size()) < 0.1)
sfweight = clipper.SFweight_spline_interface(
def exercise_restrained_refinement(options):
import random
random.seed(1)
flex.set_random_seed(1)
xs0 = smtbx.development.random_xray_structure(
sgtbx.space_group_info('P1'),
n_scatterers=options.n_scatterers,
elements="random")
for sc in xs0.scatterers():
sc.flags.set_grad_site(True)
sc0 = xs0.scatterers()
uc = xs0.unit_cell()
mi = xs0.build_miller_set(anomalous_flag=False, d_min=options.resolution)
fo_sq = mi.structure_factors_from_scatterers(
xs0, algorithm="direct").f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))
i, j, k, l = random.sample(xrange(options.n_scatterers), 4)
bond_proxies = geometry_restraints.shared_bond_simple_proxy()
w = 1e9
d_ij = uc.distance(sc0[i].site, sc0[j].site) * 0.8
bond_proxies.append(
geom.bond_simple_proxy(i_seqs=(i, j), distance_ideal=d_ij, weight=w))
d_jk = uc.distance(sc0[j].site, sc0[k].site) * 0.85
bond_proxies.append(
geom.bond_simple_proxy(i_seqs=(j, k), distance_ideal=d_jk, weight=w))
d_ki = min(
uc.distance(sc0[k].site, sc0[i].site) * 0.9, (d_ij + d_jk) * 0.8)
bond_proxies.append(
geom.bond_simple_proxy(i_seqs=(k, i), distance_ideal=d_ki, weight=w))
d_jl = uc.distance(sc0[j].site, sc0[l].site) * 0.9
bond_proxies.append(
geom.bond_simple_proxy(i_seqs=(j, l), distance_ideal=d_jl, weight=w))
d_lk = min(
uc.distance(sc0[l].site, sc0[k].site) * 0.8, 0.75 * (d_jk + d_jl))
bond_proxies.append(
geom.bond_simple_proxy(i_seqs=(l, k), distance_ideal=d_jl, weight=w))
restraints_manager = restraints.manager(bond_proxies=bond_proxies)
xs1 = xs0.deep_copy_scatterers()
xs1.shake_sites_in_place(rms_difference=0.1)
def ls_problem():
xs = xs1.deep_copy_scatterers()
reparametrisation = constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=smtbx.utils.connectivity_table(xs),
temperature=20)
return least_squares.crystallographic_ls(
fo_sq.as_xray_observations(),
reparametrisation=reparametrisation,
restraints_manager=restraints_manager)
gradient_threshold, step_threshold = 1e-6, 1e-6
eps = 5e-3
ls = ls_problem()
t = wall_clock_time()
cycles = normal_eqns_solving.naive_iterations(
ls,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold,
track_all=True)
if options.verbose:
print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles,
t.elapsed())
sc = ls.xray_structure.scatterers()
for p in bond_proxies:
d = uc.distance(*[sc[i_pair].site for i_pair in p.i_seqs])
assert approx_equal(d, p.distance_ideal, eps)
ls = ls_problem()
t = wall_clock_time()
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold,
tau=1e-3,
track_all=True)
if options.verbose:
print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles,
t.elapsed())
sc = ls.xray_structure.scatterers()
sc = ls.xray_structure.scatterers()
for p in bond_proxies:
d = uc.distance(*[sc[i].site for i in p.i_seqs])
assert approx_equal(d, p.distance_ideal, eps)
def try_den_weight_cartesian(self, grid_pair):
local_seed = int(self.random_seed+grid_pair[1])
flex.set_random_seed(value=local_seed)
random.seed(local_seed)
self.fmodels.fmodel_xray().xray_structure.replace_scatterers(
self.save_scatterers_local.deep_copy())
self.fmodels.update_xray_structure(
xray_structure = self.fmodels.fmodel_xray().xray_structure,
update_f_calc = True)
utils.assert_xray_structures_equal(
x1 = self.fmodels.fmodel_xray().xray_structure,
x2 = self.model.xray_structure)
gamma_local = grid_pair[0]
weight_local = grid_pair[1]
self.model.restraints_manager.geometry.\
den_manager.gamma = gamma_local
self.model.restraints_manager.geometry.\
den_manager.weight = weight_local
cycle = 0
self.model.restraints_manager.geometry.\
den_manager.current_cycle = cycle+1
num_den_cycles = self.model.restraints_manager.geometry.\
den_manager.num_cycles
if self.params.den.optimize and \
self.nproc != Auto and \
self.nproc > 1:
local_log = sys.stdout
elif self.params.den.optimize and \
self.nproc == 1:
if self.verbose:
local_log = self.log
else:
local_log = StringIO()
else:
local_log = self.log
print >> self.log, " ...trying gamma %f, weight %f" % (
gamma_local, weight_local)
while cycle < num_den_cycles:
print >> local_log, "DEN cycle %s" % (cycle+1)
r_free = self.fmodels.fmodel_xray().r_free()
print >> local_log, "rfree at start of SA cycle: %f" % r_free
simulated_annealing.manager(
params = self.params.simulated_annealing,
target_weights = self.target_weights,
macro_cycle = self.macro_cycle,
h_params = self.params.hydrogens,
fmodels = self.fmodels,
model = self.model,
all_params = self.params,
out = local_log)
if self.params.den.bulk_solvent_and_scale:
self.bulk_solvent_and_scale(log=local_log)
if self.params.den.refine_adp:
self.adp_refinement(log=local_log)
self.model.restraints_manager.geometry.update_dihedral_ncs_restraints(
sites_cart=self.model.xray_structure.sites_cart(),
pdb_hierarchy=self.model.pdb_hierarchy(sync_with_xray_structure=True),
log=local_log)
cycle += 1
self.model.restraints_manager.geometry.\
den_manager.current_cycle += 1
r_free = self.fmodels.fmodel_xray().r_free()
print >> local_log, "rfree at end of SA cycle: %f" % r_free
r_free = self.fmodels.fmodel_xray().r_free()
step_xray_structure = self.fmodels.fmodel_xray().\
xray_structure.deep_copy_scatterers().scatterers()
step_eq_distances = self.model.restraints_manager.geometry.\
den_manager.get_current_eq_distances()
return (gamma_local,
weight_local,
r_free,
step_xray_structure,
step_eq_distances)
def exercise_6_instantiate_consistency(symbol="C 2"):
random.seed(0)
flex.set_random_seed(0)
for scale in [1.e-4, 1.0, 1.e+4]:
for k_sol in [0, 0.3]:
for b_sol in [0, 50]:
for set_h_occ_to_zero in [True, False]:
for update_f_part1 in [True, False]:
for apply_scale_to in ["f_obs", "f_model"]:
# Simulate Fobs START
x = random_structure.xray_structure(
space_group_info=sgtbx.space_group_info(
symbol=symbol),
elements=(("O", "N", "C") * 3 + ("H", ) * 10),
volume_per_atom=50,
min_distance=3,
general_positions_only=True,
random_u_iso=True,
random_occupancy=False)
x.scattering_type_registry(table="wk1995")
x.set_occupancies(value=0.8,
selection=x.hd_selection())
f_calc = x.structure_factors(d_min=2.0).f_calc()
mask_manager = mmtbx.masks.manager(
miller_array=f_calc)
f_mask = mask_manager.shell_f_masks(
xray_structure=x)[0]
assert flex.mean(abs(f_mask).data()) > 0
b_cart = adptbx.random_traceless_symmetry_constrained_b_cart(
crystal_symmetry=x.crystal_symmetry())
u_star = adptbx.u_cart_as_u_star(
x.unit_cell(), adptbx.b_as_u(b_cart))
k_anisotropic = mmtbx.f_model.ext.k_anisotropic(
f_calc.indices(), u_star)
ss = 1. / flex.pow2(
f_calc.d_spacings().data()) / 4.
k_mask = mmtbx.f_model.ext.k_mask(ss, k_sol, b_sol)
if (apply_scale_to == "f_model"):
k_isotropic = flex.double(
f_calc.data().size(), scale)
else:
k_isotropic = flex.double(
f_calc.data().size(), 1)
f_model_data = scale * k_anisotropic * (
f_calc.data() + k_mask * f_mask.data())
f_model = f_calc.customized_copy(data=f_model_data)
f_obs = abs(f_model)
if (apply_scale_to == "f_obs"):
f_obs = f_obs.customized_copy(
data=f_obs.data() * scale)
r_free_flags = f_obs.generate_r_free_flags()
# Simulate Fobs END
if (set_h_occ_to_zero):
x.set_occupancies(value=0.0,
selection=x.hd_selection())
x.shake_sites_in_place(mean_distance=5)
sel = x.random_remove_sites_selection(fraction=0.3)
x = x.select(sel)
fmodel = mmtbx.f_model.manager(
xray_structure=x,
f_obs=f_obs,
r_free_flags=r_free_flags)
fmodel.update_all_scales(
fast=True,
show=False,
update_f_part1=update_f_part1)
f_part1_data = fmodel.f_calc().data(
) * flex.random_double(
fmodel.f_calc().data().size())
f_part1 = fmodel.f_calc().customized_copy(
data=f_part1_data)
fmodel.update(f_part1=f_part1)
r1 = fmodel.r_work()
#
zero = fmodel.f_calc().customized_copy(
data=fmodel.f_calc().data() * 0)
fmodel_dc = mmtbx.f_model.manager(
f_obs=fmodel.f_obs(),
r_free_flags=fmodel.r_free_flags(),
k_isotropic=fmodel.k_isotropic(),
k_anisotropic=fmodel.k_anisotropic(),
f_calc=fmodel.f_model_no_scales(),
f_part1=fmodel.f_part1(),
f_part2=fmodel.f_part2(),
f_mask=zero)
r2 = fmodel_dc.r_work()
if (0):
print("r1=%8.6f r2=%8.6f fp1=%6.3f fp2=%6.3f fc=%6.3f"%(r1, r2,
flex.mean(abs(fmodel.f_part1()).data()), \
flex.mean(abs(fmodel.f_part2()).data()), \
flex.mean(abs(fmodel.f_calc()).data())), \
"set_h_occ_to_zero=", set_h_occ_to_zero,\
"update_f_part1=", update_f_part1)
assert approx_equal(r1, r2), [r1, r2]
import math, sys, random
from libtbx.test_utils import approx_equal
from libtbx.utils import format_cpu_times
import random, copy
from cctbx import sgtbx
from cctbx import adptbx
from cctbx.development import random_structure
import scitbx.math
from mmtbx.tls import tools
from mmtbx_tls_ext import *
from six.moves import range
if (1): # fixed random seed to avoid rare failures
random.seed(0)
flex.set_random_seed(0)
def branch_3_mn():
m = None
n = None
small = 1.e-15
for i in range(10000):
m2 = []
n2 = []
for i in range(4):
r1 = random.random()
r2 = random.random()
r3 = random.random()
if (r3 > 0.1):
r1 = r2
def exercise_6_instantiate_consistency(symbol = "C 2"):
random.seed(0)
flex.set_random_seed(0)
for scale in [1.e-4, 1.0, 1.e+4]:
for k_sol in [0, 0.3]:
for b_sol in [0, 50]:
for set_h_occ_to_zero in [True, False]:
for update_f_part1 in [True, False]:
for apply_scale_to in ["f_obs", "f_model"]:
# Simulate Fobs START
x = random_structure.xray_structure(
space_group_info = sgtbx.space_group_info(symbol=symbol),
elements =(("O","N","C")*3+("H",)*10),
volume_per_atom = 50,
min_distance = 3,
general_positions_only = True,
random_u_iso = True,
random_occupancy = False)
x.scattering_type_registry(table="wk1995")
x.set_occupancies(value=0.8, selection = x.hd_selection())
f_calc = x.structure_factors(d_min = 2.0).f_calc()
mask_manager = mmtbx.masks.manager(miller_array = f_calc)
f_mask = mask_manager.shell_f_masks(xray_structure = x)[0]
assert flex.mean(abs(f_mask).data()) > 0
b_cart=adptbx.random_traceless_symmetry_constrained_b_cart(
crystal_symmetry=x.crystal_symmetry())
u_star = adptbx.u_cart_as_u_star(x.unit_cell(), adptbx.b_as_u(b_cart))
k_anisotropic = mmtbx.f_model.ext.k_anisotropic(f_calc.indices(), u_star)
ss = 1./flex.pow2(f_calc.d_spacings().data()) / 4.
k_mask = mmtbx.f_model.ext.k_mask(ss, k_sol, b_sol)
if(apply_scale_to=="f_model"):
k_isotropic = flex.double(f_calc.data().size(), scale)
else:
k_isotropic = flex.double(f_calc.data().size(), 1)
f_model_data = scale*k_anisotropic*(f_calc.data()+k_mask*f_mask.data())
f_model = f_calc.customized_copy(data = f_model_data)
f_obs = abs(f_model)
if(apply_scale_to=="f_obs"):
f_obs = f_obs.customized_copy(data = f_obs.data()*scale)
r_free_flags = f_obs.generate_r_free_flags()
# Simulate Fobs END
if(set_h_occ_to_zero):
x.set_occupancies(value=0.0, selection = x.hd_selection())
x.shake_sites_in_place(mean_distance=5)
sel = x.random_remove_sites_selection(fraction=0.3)
x = x.select(sel)
fmodel = mmtbx.f_model.manager(
xray_structure = x,
f_obs = f_obs,
r_free_flags = r_free_flags)
fmodel.update_all_scales(fast=True, show=False,
update_f_part1=update_f_part1)
f_part1_data = fmodel.f_calc().data()*flex.random_double(
fmodel.f_calc().data().size())
f_part1 = fmodel.f_calc().customized_copy(data = f_part1_data)
fmodel.update(f_part1 = f_part1)
r1 = fmodel.r_work()
#
zero=fmodel.f_calc().customized_copy(data=fmodel.f_calc().data()*0)
fmodel_dc = mmtbx.f_model.manager(
f_obs = fmodel.f_obs(),
r_free_flags = fmodel.r_free_flags(),
k_isotropic = fmodel.k_isotropic(),
k_anisotropic = fmodel.k_anisotropic(),
f_calc = fmodel.f_model_no_scales(),
f_part1 = fmodel.f_part1(),
f_part2 = fmodel.f_part2(),
f_mask = zero)
r2 = fmodel_dc.r_work()
if(0):
print "r1=%8.6f r2=%8.6f fp1=%6.3f fp2=%6.3f fc=%6.3f"%(r1, r2,
flex.mean(abs(fmodel.f_part1()).data()), \
flex.mean(abs(fmodel.f_part2()).data()), \
flex.mean(abs(fmodel.f_calc()).data())), \
"set_h_occ_to_zero=", set_h_occ_to_zero,\
"update_f_part1=", update_f_part1
assert approx_equal(r1, r2), [r1, r2]
