def check_cb_op_perm(cb_op, perm):
mi_cb = cb_op.apply(ra.miller_indices)
miis = flex.random_permutation(size=ra.miller_indices.size())[:2]
k = cb_op.apply(ra.miller_indices.select(miis))
matches = miller.match_indices(k, ra.miller_indices)
pairs = matches.pairs()
assert pairs.column(0).all_eq(flex.size_t_range(k.size()))
miis_cb = pairs.column(1)
assert perm.select(miis).all_eq(miis_cb)
def check_cb_op_perm(cb_op, perm): mi_cb = cb_op.apply(ra.miller_indices) miis = flex.random_permutation(size=ra.miller_indices.size())[:2] k = cb_op.apply(ra.miller_indices.select(miis)) matches = miller.match_indices(k, ra.miller_indices) pairs = matches.pairs() assert pairs.column(0).all_eq(flex.size_t_range(k.size())) miis_cb = pairs.column(1) assert perm.select(miis).all_eq(miis_cb)
def exercise_optimise_shelxl_weights():
def calc_goof(fo2, fc, w, k, n_params):
fc2 = fc.as_intensity_array()
w = w(fo2.data(), fo2.sigmas(), fc2.data(), k)
return math.sqrt(
flex.sum(w * flex.pow2(fo2.data() - k * fc2.data())) /
(fo2.size() - n_params))
xs = smtbx.development.sucrose()
k = 0.05 + 10 * flex.random_double()
fc = xs.structure_factors(anomalous_flag=False, d_min=0.7).f_calc()
fo = fc.as_amplitude_array()
fo = fo.customized_copy(data=fo.data() * math.sqrt(k))
fo = fo.customized_copy(sigmas=0.03 * fo.data())
sigmas = fo.sigmas()
for i in range(fo.size()):
fo.data()[i] += 2 * scitbx.random.variate(
scitbx.random.normal_distribution(sigma=sigmas[i]))() \
+ 0.5*random.random()
fo2 = fo.as_intensity_array()
fc2 = fc.as_intensity_array()
w = least_squares.mainstream_shelx_weighting(a=0.1)
s = calc_goof(fo2, fc, w, k, xs.n_parameters())
w2 = w.optimise_parameters(fo2, fc2, k, xs.n_parameters())
s2 = calc_goof(fo2, fc, w2, k, xs.n_parameters())
# sort data and setup binning by fc/fc_max
fc_sq = fc.as_intensity_array()
fc_sq_over_fc_sq_max = fc_sq.data() / flex.max(fc_sq.data())
permutation = flex.sort_permutation(fc_sq_over_fc_sq_max)
fc_sq_over_fc_sq_max = fc_sq.customized_copy(
data=fc_sq_over_fc_sq_max).select(permutation)
fc_sq = fc_sq.select(permutation)
fo_sq = fo2.select(permutation)
n_bins = 10
bin_max = 0
bin_limits = flex.size_t(1, 0)
bin_count = flex.size_t()
for i in range(n_bins):
bin_limits.append(int(math.ceil((i + 1) * fc_sq.size() / n_bins)))
bin_count.append(bin_limits[i + 1] - bin_limits[i])
goofs_w = flex.double()
goofs_w2 = flex.double()
for i_bin in range(n_bins):
sel = flex.size_t_range(bin_limits[i_bin], bin_limits[i_bin + 1])
goofs_w2.append(
calc_goof(fo_sq.select(sel), fc_sq.select(sel), w2, k,
xs.n_parameters()))
goofs_w.append(
calc_goof(fo_sq.select(sel), fc_sq.select(sel), w, k,
xs.n_parameters()))
a = flex.mean_and_variance(goofs_w).unweighted_sample_variance()
b = flex.mean_and_variance(goofs_w2).unweighted_sample_variance()
assert a > b or abs(1 - s) > abs(1 - s2)
assert a > b # flat analysis of variance
assert abs(1 - s) > abs(1 - s2) # GooF close to 1
def e_pot(O, sites_moved):
if ( O.last_sites_moved is None
or O.last_sites_moved.id() != sites_moved.id()):
O.last_sites_moved = sites_moved
xs = O.fmodels.fmodel_xray().xray_structure
assert len(sites_moved) == xs.scatterers().size()
sites_cart = sites_moved
xs.set_sites_cart(sites_cart=sites_cart)
O.fmodels.update_xray_structure(update_f_calc=True)
xs.scatterers().flags_set_grads(state=False)
xs.scatterers().flags_set_grad_site(
iselection=flex.size_t_range(xs.scatterers().size()))
expected_g_size = len(sites_moved) * 3
if (O.xray_weight_factor is not None):
tg = O.fmodels.target_and_gradients(
weights=O.weights,
compute_gradients=True)
O.f = tg.target() * O.xray_weight_factor
O.g = tg.gradients() * O.xray_weight_factor
assert O.g.size() == expected_g_size
else:
O.f = 0.
O.g = flex.double(expected_g_size, 0)
if (O.reduced_geo_manager is None):
reduced_geo_energies = None
else:
reduced_geo_energies = O.reduced_geo_manager.energies_sites(
sites_cart=sites_cart,
compute_gradients=True)
other_energies = O.model.restraints_manager.energies_sites(
sites_cart=sites_cart,
geometry_flags=cctbx.geometry_restraints.flags.flags(nonbonded=True),
custom_nonbonded_function=O.custom_nonbonded_function,
compute_gradients=True)
nfw = other_energies.normalization_factor * O.weights.w
O.f += other_energies.target * O.weights.w
gg = other_energies.gradients * O.weights.w
if (reduced_geo_energies is not None):
O.f += reduced_geo_energies.target * nfw
gg += reduced_geo_energies.gradients * nfw
assert nfw != 0
scale = 1 / nfw
O.last_grms = group_args(
geo=scale*flex.mean_sq(gg.as_double())**0.5,
xray=scale*flex.mean_sq(O.g)**0.5,
real_or_xray="xray")
xray.minimization.add_gradients(
scatterers=xs.scatterers(),
xray_gradients=O.g,
site_gradients=gg)
O.f *= scale
O.g *= scale
O.last_grms.total = flex.mean_sq(O.g)**0.5
O.g = flex.vec3_double(O.g)
return O.f
def e_pot(O, sites_moved):
if (O.last_sites_moved is None
or O.last_sites_moved.id() != sites_moved.id()):
O.last_sites_moved = sites_moved
xs = O.fmodels.fmodel_xray().xray_structure
assert len(sites_moved) == xs.scatterers().size()
sites_cart = sites_moved
xs.set_sites_cart(sites_cart=sites_cart)
O.fmodels.update_xray_structure(update_f_calc=True)
xs.scatterers().flags_set_grads(state=False)
xs.scatterers().flags_set_grad_site(
iselection=flex.size_t_range(xs.scatterers().size()))
expected_g_size = len(sites_moved) * 3
if (O.xray_weight_factor is not None):
tg = O.fmodels.target_and_gradients(weights=O.weights,
compute_gradients=True)
O.f = tg.target() * O.xray_weight_factor
O.g = tg.gradients() * O.xray_weight_factor
assert O.g.size() == expected_g_size
else:
O.f = 0.
O.g = flex.double(expected_g_size, 0)
if (O.reduced_geo_manager is None):
reduced_geo_energies = None
else:
reduced_geo_energies = O.reduced_geo_manager.energies_sites(
sites_cart=sites_cart, compute_gradients=True)
other_energies = O.model.restraints_manager.energies_sites(
sites_cart=sites_cart,
geometry_flags=cctbx.geometry_restraints.flags.flags(
nonbonded=True),
custom_nonbonded_function=O.custom_nonbonded_function,
compute_gradients=True)
nfw = other_energies.normalization_factor * O.weights.w
O.f += other_energies.target * O.weights.w
gg = other_energies.gradients * O.weights.w
if (reduced_geo_energies is not None):
O.f += reduced_geo_energies.target * nfw
gg += reduced_geo_energies.gradients * nfw
assert nfw != 0
scale = 1 / nfw
O.last_grms = group_args(geo=scale *
flex.mean_sq(gg.as_double())**0.5,
xray=scale * flex.mean_sq(O.g)**0.5,
real_or_xray="xray")
xray.minimization.add_gradients(scatterers=xs.scatterers(),
xray_gradients=O.g,
site_gradients=gg)
O.f *= scale
O.g *= scale
O.last_grms.total = flex.mean_sq(O.g)**0.5
O.g = flex.vec3_double(O.g)
return O.f
def exercise_optimise_shelxl_weights():
def calc_goof(fo2, fc, w, k, n_params):
fc2 = fc.as_intensity_array()
w = w(fo2.data(), fo2.sigmas(), fc2.data(), k)
return math.sqrt(flex.sum(
w * flex.pow2(fo2.data() - k*fc2.data()))/(fo2.size() - n_params))
xs = smtbx.development.sucrose()
k = 0.05 + 10 * flex.random_double()
fc = xs.structure_factors(anomalous_flag=False, d_min=0.7).f_calc()
fo = fc.as_amplitude_array()
fo = fo.customized_copy(data=fo.data()*math.sqrt(k))
fo = fo.customized_copy(sigmas=0.03*fo.data())
sigmas = fo.sigmas()
for i in range(fo.size()):
fo.data()[i] += 2 * scitbx.random.variate(
scitbx.random.normal_distribution(sigma=sigmas[i]))() \
+ 0.5*random.random()
fo2 = fo.as_intensity_array()
fc2 = fc.as_intensity_array()
w = least_squares.mainstream_shelx_weighting(a=0.1)
s = calc_goof(fo2, fc, w, k, xs.n_parameters())
w2 = w.optimise_parameters(fo2, fc2, k, xs.n_parameters())
s2 = calc_goof(fo2, fc, w2, k, xs.n_parameters())
# sort data and setup binning by fc/fc_max
fc_sq = fc.as_intensity_array()
fc_sq_over_fc_sq_max = fc_sq.data()/flex.max(fc_sq.data())
permutation = flex.sort_permutation(fc_sq_over_fc_sq_max)
fc_sq_over_fc_sq_max = fc_sq.customized_copy(
data=fc_sq_over_fc_sq_max).select(permutation)
fc_sq = fc_sq.select(permutation)
fo_sq = fo2.select(permutation)
n_bins = 10
bin_max = 0
bin_limits = flex.size_t(1, 0)
bin_count = flex.size_t()
for i in range(n_bins):
bin_limits.append(int(math.ceil((i+1) * fc_sq.size()/n_bins)))
bin_count.append(bin_limits[i+1] - bin_limits[i])
goofs_w = flex.double()
goofs_w2 = flex.double()
for i_bin in range(n_bins):
sel = flex.size_t_range(bin_limits[i_bin], bin_limits[i_bin+1])
goofs_w2.append(calc_goof(fo_sq.select(sel),
fc_sq.select(sel),
w2, k, xs.n_parameters()))
goofs_w.append(calc_goof(fo_sq.select(sel),
fc_sq.select(sel),
w, k, xs.n_parameters()))
a = flex.mean_and_variance(goofs_w).unweighted_sample_variance()
b = flex.mean_and_variance(goofs_w2).unweighted_sample_variance()
assert a > b or abs(1-s) > abs(1-s2)
assert a > b # flat analysis of variance
assert abs(1-s) > abs(1-s2) # GooF close to 1
def optimise_parameters(self, fo_sq, fc_sq,
scale_factor, n_independent_params):
""" Find optimal values of a and b that give a flat analysis of the variance
when binned by Fc/max(Fc), and a goodness of fit close to 1.
This is done in a grid search fashion similar to Shelxl.
self is not modified in place; instead a new instance of the weighting
scheme is returned.
It is intended that f_calc should already contain the contribution from
f_mask (if a solvent mask is used).
"""
assert fc_sq.is_xray_intensity_array()
weighting = ext.mainstream_shelx_weighting(a=self.a, b=self.b)
def compute_chi_sq(fo_sq, fc_sq, a,b):
weighting.a = a
weighting.b = b
weights = weighting(
fo_sq.data(), fo_sq.sigmas(), fc_sq.data(), scale_factor)
return (flex.sum(
weights * flex.pow2(fo_sq.data() - scale_factor * fc_sq.data())))
fo_sq = fo_sq.deep_copy()
fo_sq.data().set_selected(fo_sq.data() < 0, 0)
fo2 = fo_sq.data().deep_copy()
fo2 /= scale_factor
sigmas = fo_sq.sigmas() / scale_factor
sigmas_sq = flex.pow2(sigmas)
fc2 = fc_sq.data()
# determine starting values for a and b, formulae taken from shelxl code
p = (fo2 + 2 * fc2)/3
p_sq = flex.pow2(p)
x = flex.sum((flex.pow2(fo2-fc2)-sigmas) * (p_sq/sigmas_sq))
y = flex.sum( flex.pow2(p_sq)/sigmas_sq)
z = flex.sum(p)
start_a = math.sqrt(max(0.0001, 0.64*x/max(1e-8, y)))
start_b = 0.5 * z * start_a**2 /fo_sq.size()
a_step = 0.2 * start_a
b_step = 0.4 * start_b
# sort data and setup binning by fc/fc_max
fc_sq_over_fc_sq_max = fc_sq.data()/flex.max(fc_sq.data())
permutation = flex.sort_permutation(fc_sq_over_fc_sq_max)
fc_sq_over_fc_sq_max = fc_sq.customized_copy(
data=fc_sq_over_fc_sq_max).select(permutation)
fc_sq = fc_sq.select(permutation)
fo_sq = fo_sq.select(permutation)
n_bins = 10
bin_max = 0
bin_limits = flex.size_t(1, 0)
bin_count = flex.size_t()
for i in range(n_bins):
bin_limits.append(int(math.ceil((i+1) * fc_sq.size()/n_bins)))
bin_count.append(bin_limits[i+1] - bin_limits[i])
n = fo_sq.size()//(fo_sq.size()-n_independent_params)
# search on a 9x9 grid to determine best values of a and b
gridding = flex.grid(9,9)
while (a_step > 1e-4 and b_step > 5e-3):
tmp = flex.double(gridding, 0)
binned_chi_sq = [tmp.deep_copy() for i in range(n_bins)]
start_a = max(start_a, 4*a_step) - 4*a_step
start_b = max(start_b, 4*b_step) - 4*b_step
for i_bin in range(n_bins):
sel = flex.size_t_range(bin_limits[i_bin], bin_limits[i_bin+1])
fc2 = fc_sq.select(sel)
fo2 = fo_sq.select(sel)
b = start_b
for j in range(9):
a = start_a
b += b_step
for k in range(9):
a += a_step
binned_chi_sq[i_bin][j,k] += compute_chi_sq(fo2, fc2, a, b)
min_variance = 9e9
j_min, k_min = (0, 0)
for j in range(9):
for k in range(9):
variance = 0
for i_bin in range(n_bins):
if bin_count[i_bin] == 0: continue
goof = math.sqrt(binned_chi_sq[i_bin][j,k]*n/bin_count[i_bin])
variance += (goof-1)**2
min_variance = min(variance, min_variance)
if variance == min_variance:
j_min = j
k_min = k
start_a += k_min*a_step
start_b += j_min*b_step
if k_min == 8:
a_step *= 2
continue
elif k_min != 0:
a_step /= 4
if j_min == 8:
b_step *= 2
continue
elif j_min != 0:
b_step /=4
if start_a <= 1e-4: a_step /= 4
if start_b <= 1e-3: b_step /= 4
if start_a > 0.2:
start_a = 0.2
start_b = 0
weighting.a = start_a
weighting.b = start_b
return weighting
def exercise(verbose=0):
distance_ideal = 1.8
default_vdw_distance = 3.6
vdw_1_4_factor = 3.5 / 3.6
sites_cart_manual = flex.vec3_double([(1, 3, 0), (2, 3, 0), (3, 2, 0),
(3, 1, 0), (4, 1, 0), (3, 4, 0),
(4, 3, 0), (5, 3, 0), (6, 2, 0),
(7, 2, 0), (8, 3, 0), (7, 4, 0),
(6, 4, 0), (7, 5, 0), (6, 6, 0),
(8, 6, 0)])
bond_proxies = geometry_restraints.bond_sorted_asu_proxies(
asu_mappings=None)
for i_seqs in [(0, 1), (1, 2), (2, 3), (3, 4), (1, 5), (2, 6), (5, 6),
(6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12),
(12, 7), (11, 13), (13, 14), (14, 15), (15, 13)]:
bond_proxies.process(
geometry_restraints.bond_simple_proxy(
i_seqs=i_seqs, distance_ideal=distance_ideal, weight=100))
angle_proxies = geometry_restraints.shared_angle_proxy()
for i_seqs, angle_ideal in [[(0, 1, 2), 135], [(0, 1, 5), 135],
[(1, 2, 3), 135], [(3, 2, 6), 135],
[(2, 3, 4), 120], [(1, 2, 6), 90],
[(2, 6, 5), 90], [(6, 5, 1), 90],
[(5, 1, 2), 90], [(2, 6, 7), 135],
[(5, 6, 7), 135], [(6, 7, 8), 120],
[(6, 7, 12), 120], [(7, 8, 9), 120],
[(8, 9, 10), 120], [(9, 10, 11), 120],
[(10, 11, 12), 120], [(11, 12, 7), 120],
[(12, 7, 8), 120], [(10, 11, 13), 120],
[(12, 11, 13), 120], [(11, 13, 15), 150],
[(11, 13, 14), 150], [(13, 15, 14), 60],
[(15, 14, 13), 60], [(14, 13, 15), 60]]:
angle_proxies.append(
geometry_restraints.angle_proxy(i_seqs=i_seqs,
angle_ideal=angle_ideal,
weight=1))
if (0 or verbose):
dump_pdb(file_name="manual.pdb", sites_cart=sites_cart_manual)
for traditional_convergence_test in [True, False]:
for sites_cart_selection in [True, False]:
sites_cart = sites_cart_manual.deep_copy()
if sites_cart_selection:
sites_cart_selection = flex.bool(sites_cart.size(), True)
sites_cart_selection[1] = False
assert bond_proxies.asu.size() == 0
bond_params_table = geometry_restraints.extract_bond_params(
n_seq=sites_cart.size(),
bond_simple_proxies=bond_proxies.simple)
manager = geometry_restraints.manager.manager(
bond_params_table=bond_params_table,
angle_proxies=angle_proxies)
minimized = geometry_restraints.lbfgs.lbfgs(
sites_cart=sites_cart,
geometry_restraints_manager=manager,
lbfgs_termination_params=scitbx.lbfgs.termination_parameters(
traditional_convergence_test=traditional_convergence_test,
drop_convergence_test_max_drop_eps=1.e-20,
drop_convergence_test_iteration_coefficient=1,
max_iterations=1000),
sites_cart_selection=sites_cart_selection,
)
assert minimized.minimizer.iter() > 100
sites_cart_minimized_1 = sites_cart.deep_copy()
if (0 or verbose):
dump_pdb(file_name="minimized_1.pdb",
sites_cart=sites_cart_minimized_1)
bond_deltas = geometry_restraints.bond_deltas(
sites_cart=sites_cart_minimized_1, proxies=bond_proxies.simple)
angle_deltas = geometry_restraints.angle_deltas(
sites_cart=sites_cart_minimized_1, proxies=angle_proxies)
if (0 or verbose):
for proxy, delta in zip(bond_proxies.simple, bond_deltas):
print "bond:", proxy.i_seqs, delta
for proxy, delta in zip(angle_proxies, angle_deltas):
print "angle:", proxy.i_seqs, delta
assert is_below_limit(value=flex.max(flex.abs(bond_deltas)),
limit=0,
eps=1.e-6)
assert is_below_limit(value=flex.max(flex.abs(angle_deltas)),
limit=0,
eps=2.e-6)
sites_cart += matrix.col((1, 1, 0)) - matrix.col(sites_cart.min())
unit_cell_lengths = list(
matrix.col(sites_cart.max()) + matrix.col((1, -1.2, 4)))
unit_cell_lengths[1] *= 2
unit_cell_lengths[2] *= 2
xray_structure = xray.structure(crystal_symmetry=crystal.symmetry(
unit_cell=unit_cell_lengths, space_group_symbol="P112"))
for serial, site in zip(count(1), sites_cart):
xray_structure.add_scatterer(
xray.scatterer(
label="C%02d" % serial,
site=xray_structure.unit_cell().fractionalize(site)))
if (0 or verbose):
xray_structure.show_summary().show_scatterers()
p1_structure = (xray_structure.apply_shift(
(-.5, -.5, 0)).expand_to_p1().apply_shift((.5, .5, 0)))
for shift in [(1, 0, 0), (0, 1, 0), (0, 0, 1)]:
p1_structure.add_scatterers(
p1_structure.apply_shift(shift).scatterers())
if (0 or verbose):
open("p1_structure.pdb", "w").write(p1_structure.as_pdb_file())
nonbonded_cutoff = 6.5
asu_mappings = xray_structure.asu_mappings(
buffer_thickness=nonbonded_cutoff)
bond_asu_table = crystal.pair_asu_table(asu_mappings=asu_mappings)
geometry_restraints.add_pairs(bond_asu_table, bond_proxies.simple)
shell_asu_tables = crystal.coordination_sequences.shell_asu_tables(
pair_asu_table=bond_asu_table, max_shell=3)
shell_sym_tables = [
shell_asu_table.extract_pair_sym_table()
for shell_asu_table in shell_asu_tables
]
bond_params_table = geometry_restraints.extract_bond_params(
n_seq=sites_cart.size(), bond_simple_proxies=bond_proxies.simple)
atom_energy_types = flex.std_string(sites_cart.size(), "Default")
nonbonded_params = geometry_restraints.nonbonded_params(
factor_1_4_interactions=vdw_1_4_factor,
const_shrink_1_4_interactions=0,
default_distance=default_vdw_distance)
nonbonded_params.distance_table.setdefault(
"Default")["Default"] = default_vdw_distance
pair_proxies = geometry_restraints.pair_proxies(
bond_params_table=bond_params_table,
shell_asu_tables=shell_asu_tables,
model_indices=None,
conformer_indices=None,
nonbonded_params=nonbonded_params,
nonbonded_types=atom_energy_types,
nonbonded_distance_cutoff_plus_buffer=nonbonded_cutoff)
if (0 or verbose):
print "pair_proxies.bond_proxies.n_total():", \
pair_proxies.bond_proxies.n_total(),
print "simple:", pair_proxies.bond_proxies.simple.size(),
print "sym:", pair_proxies.bond_proxies.asu.size()
print "pair_proxies.nonbonded_proxies.n_total():", \
pair_proxies.nonbonded_proxies.n_total(),
print "simple:", pair_proxies.nonbonded_proxies.simple.size(),
print "sym:", pair_proxies.nonbonded_proxies.asu.size()
print "min_distance_nonbonded: %.2f" % flex.min(
geometry_restraints.nonbonded_deltas(
sites_cart=sites_cart,
sorted_asu_proxies=pair_proxies.nonbonded_proxies))
s = StringIO()
pair_proxies.bond_proxies.show_histogram_of_model_distances(
sites_cart=sites_cart, f=s, prefix="[]")
assert s.getvalue().splitlines()[0] == "[]Histogram of bond lengths:"
assert s.getvalue().splitlines()[5].startswith("[] 1.80 - 1.80:")
s = StringIO()
pair_proxies.bond_proxies.show_histogram_of_deltas(sites_cart=sites_cart,
f=s,
prefix="][")
assert s.getvalue().splitlines()[0] == "][Histogram of bond deltas:"
assert s.getvalue().splitlines()[5].startswith("][ 0.000 - 0.000:")
s = StringIO()
pair_proxies.bond_proxies.show_sorted(by_value="residual",
sites_cart=sites_cart,
max_items=3,
f=s,
prefix=":;")
l = s.getvalue().splitlines()
assert l[0] == ":;Bond restraints: 18"
assert l[1] == ":;Sorted by residual:"
assert l[2].startswith(":;bond ")
assert l[3].startswith(":; ")
assert l[4] == ":; ideal model delta sigma weight residual"
for i in [5, -2]:
assert l[i].startswith(":; 1.800 1.800 ")
assert l[-1] == ":;... (remaining 15 not shown)"
s = StringIO()
pair_proxies.nonbonded_proxies.show_histogram_of_model_distances(
sites_cart=sites_cart, f=s, prefix="]^")
assert not show_diff(
s.getvalue(), """\
]^Histogram of nonbonded interaction distances:
]^ 2.16 - 3.03: 3
]^ 3.03 - 3.89: 12
]^ 3.89 - 4.75: 28
]^ 4.75 - 5.61: 44
]^ 5.61 - 6.48: 54
""")
s = StringIO()
pair_proxies.nonbonded_proxies.show_sorted(by_value="delta",
sites_cart=sites_cart,
max_items=7,
f=s,
prefix=">,")
assert not show_diff(s.getvalue(),
"""\
>,Nonbonded interactions: 141
>,Sorted by model distance:
>,nonbonded 15
>, 15
>, model vdw sym.op.
>, 2.164 3.600 -x+2,-y+1,z
...
>,nonbonded 4
>, 8
>, model vdw
>, 3.414 3.600
>,... (remaining 134 not shown)
""",
selections=[range(6), range(-5, 0)])
vdw_1_sticks = []
vdw_2_sticks = []
for proxy in pair_proxies.nonbonded_proxies.simple:
if (proxy.vdw_distance == default_vdw_distance):
vdw_1_sticks.append(
pml_stick(begin=sites_cart[proxy.i_seqs[0]],
end=sites_cart[proxy.i_seqs[1]]))
else:
vdw_2_sticks.append(
pml_stick(begin=sites_cart[proxy.i_seqs[0]],
end=sites_cart[proxy.i_seqs[1]]))
mps = asu_mappings.mappings()
for proxy in pair_proxies.nonbonded_proxies.asu:
if (proxy.vdw_distance == default_vdw_distance):
vdw_1_sticks.append(
pml_stick(begin=mps[proxy.i_seq][0].mapped_site(),
end=mps[proxy.j_seq][proxy.j_sym].mapped_site()))
else:
vdw_2_sticks.append(
pml_stick(begin=mps[proxy.i_seq][0].mapped_site(),
end=mps[proxy.j_seq][proxy.j_sym].mapped_site()))
if (0 or verbose):
pml_write(f=open("vdw_1.pml", "w"), label="vdw_1", sticks=vdw_1_sticks)
pml_write(f=open("vdw_2.pml", "w"), label="vdw_2", sticks=vdw_2_sticks)
#
i_pdb = count(2)
for use_crystal_symmetry in [False, True]:
if (not use_crystal_symmetry):
crystal_symmetry = None
site_symmetry_table = None
else:
crystal_symmetry = xray_structure
site_symmetry_table = xray_structure.site_symmetry_table()
for sites_cart in [
sites_cart_manual.deep_copy(),
sites_cart_minimized_1.deep_copy()
]:
manager = geometry_restraints.manager.manager(
crystal_symmetry=crystal_symmetry,
site_symmetry_table=site_symmetry_table,
nonbonded_params=nonbonded_params,
nonbonded_types=atom_energy_types,
nonbonded_function=geometry_restraints.
prolsq_repulsion_function(),
bond_params_table=bond_params_table,
shell_sym_tables=shell_sym_tables,
nonbonded_distance_cutoff=nonbonded_cutoff,
nonbonded_buffer=1,
angle_proxies=angle_proxies,
plain_pairs_radius=5)
manager = manager.select(
selection=flex.bool(sites_cart.size(), True))
manager = manager.select(iselection=flex.size_t_range(
stop=sites_cart.size()))
pair_proxies = manager.pair_proxies(sites_cart=sites_cart)
minimized = geometry_restraints.lbfgs.lbfgs(
sites_cart=sites_cart,
geometry_restraints_manager=manager,
lbfgs_termination_params=scitbx.lbfgs.termination_parameters(
max_iterations=1000))
if (0 or verbose):
minimized.final_target_result.show()
print "number of function evaluations:", minimized.minimizer.nfun(
)
print "n_updates_pair_proxies:", manager.n_updates_pair_proxies
if (not use_crystal_symmetry):
assert minimized.final_target_result.bond_residual_sum < 1.e-3
assert minimized.final_target_result.nonbonded_residual_sum < 0.1
else:
assert minimized.final_target_result.bond_residual_sum < 1.e-2
assert minimized.final_target_result.nonbonded_residual_sum < 0.1
assert minimized.final_target_result.angle_residual_sum < 1.e-3
if (0 or verbose):
pdb_file_name = "minimized_%d.pdb" % i_pdb.next()
print "Writing file:", pdb_file_name
dump_pdb(file_name=pdb_file_name, sites_cart=sites_cart)
if (manager.site_symmetry_table is None):
additional_site_symmetry_table = None
else:
additional_site_symmetry_table = sgtbx.site_symmetry_table()
assert manager.new_including_isolated_sites(
n_additional_sites=0,
site_symmetry_table=additional_site_symmetry_table,
nonbonded_types=flex.std_string()).plain_pairs_radius \
== manager.plain_pairs_radius
if (crystal_symmetry is not None):
assert len(manager.plain_pair_sym_table) == 16
if (0 or verbose):
manager.plain_pair_sym_table.show()
#
xray_structure.set_u_iso(values=flex.double([
0.77599982480241358, 0.38745781137212021, 0.20667558236418682,
0.99759840171302094, 0.8917287406687805, 0.64780251325379845,
0.24878590382983534, 0.59480621182194615, 0.58695637792905142,
0.33997130213653637, 0.51258699130743735, 0.79760289141276675,
0.39996577657875021, 0.4329328819341467, 0.70422156561726479,
0.87260110626999332
]))
class parameters:
pass
parameters.sphere_radius = 5
parameters.distance_power = 0.7
parameters.average_power = 0.9
parameters.wilson_b_weight = 1.3952
parameters.wilson_b_weight_auto = False
adp_energies = adp_restraints.energies_iso(
geometry_restraints_manager=manager,
xray_structure=xray_structure,
parameters=parameters,
wilson_b=None,
use_hd=False,
use_u_local_only=False,
compute_gradients=False,
gradients=None,
normalization=False,
collect=True)
assert adp_energies.number_of_restraints == 69
assert approx_equal(adp_energies.residual_sum, 6.24865382467)
assert adp_energies.gradients is None
assert adp_energies.u_i.size() == adp_energies.number_of_restraints
assert adp_energies.u_j.size() == adp_energies.number_of_restraints
assert adp_energies.r_ij.size() == adp_energies.number_of_restraints
for wilson_b in [None, 10, 100]:
finite_difference_gradients = flex.double()
eps = 1.e-6
for i_scatterer in xrange(xray_structure.scatterers().size()):
rs = []
for signed_eps in [eps, -eps]:
xray_structure_eps = xray_structure.deep_copy_scatterers()
xray_structure_eps.scatterers(
)[i_scatterer].u_iso += signed_eps
adp_energies = adp_restraints.energies_iso(
geometry_restraints_manager=manager,
xray_structure=xray_structure_eps,
parameters=parameters,
wilson_b=wilson_b,
use_u_local_only=False,
use_hd=False,
compute_gradients=True,
gradients=None,
normalization=False,
collect=False)
rs.append(adp_energies.residual_sum)
assert adp_energies.gradients.size() \
== xray_structure.scatterers().size()
assert adp_energies.u_i == None
assert adp_energies.u_j == None
assert adp_energies.r_ij == None
finite_difference_gradients.append((rs[0] - rs[1]) / (2 * eps))
sel = flex.bool(xray_structure.scatterers().size(), True)
xray_structure.scatterers().flags_set_grad_u_iso(sel.iselection())
adp_energies = adp_restraints.energies_iso(
geometry_restraints_manager=manager,
xray_structure=xray_structure,
parameters=parameters,
wilson_b=wilson_b,
use_u_local_only=False,
use_hd=False,
compute_gradients=True,
gradients=None,
normalization=False,
collect=False)
assert approx_equal(adp_energies.gradients,
finite_difference_gradients)
print "OK"
def exercise_affine_occupancy_parameter():
xs = xray.structure(crystal_symmetry=crystal.symmetry(
unit_cell=(), space_group_symbol='hall: P 1'),
scatterers=flex.xray_scatterer((
xray.scatterer('C0', occupancy=1),
xray.scatterer('C1', occupancy=1),
xray.scatterer('C2', occupancy=1),
xray.scatterer('C3', occupancy=1),
)))
sc = xs.scatterers()
sc.flags_set_grad_occupancy(flex.size_t_range(4))
# Two occupancies adding up to 1 (most common case of disorder)
r = constraints.ext.reparametrisation(xs.unit_cell())
occ_1 = r.add(constraints.independent_occupancy_parameter, sc[1])
occ_3 = r.add(constraints.affine_asu_occupancy_parameter,
dependee=occ_1,
a=-1,
b=1,
scatterer=sc[3])
r.finalise()
r.linearise()
assert approx_equal(occ_1.value, 1)
assert approx_equal(occ_3.value, 0)
jt0 = sparse.matrix(
1,
2,
[
{
0: 1
}, # 1st col = derivatives of occ_1
{
0: -1
}, # 2nd col = derivatives of occ_3
])
assert sparse.approx_equal(tolerance=1e-15)(r.jacobian_transpose, jt0)
# Example illustrating the instruction SUMP in SHELX 97 manual (p. 7-26)
# We disregard the issue of the special position which is orthogonal to the
# point we want to test here.
xs = xray.structure(crystal_symmetry=crystal.symmetry(
unit_cell=(), space_group_symbol='hall: P 1'),
scatterers=flex.xray_scatterer((
xray.scatterer('Na+', occupancy=1),
xray.scatterer('Ca2+', occupancy=1),
xray.scatterer('Al3+', occupancy=0.35),
xray.scatterer('K+', occupancy=0.15),
)))
sc = xs.scatterers()
sc.flags_set_grad_occupancy(flex.size_t_range(4))
# The constraints are:
# fully occupied: occ(Na+) + occ(Ca2+) + occ(Al3+) + occ(K+) = 1
# average charge +2: occ(Na+) + 2 occ(Ca2+) + 3 occ(Al3+) + occ(K+) = +2
# This can be solved as:
# occ(Na+) = occ(Al3+) - occ(K+)
# occ(Ca2+) = 1 - 2 occ(Al3+)
r = constraints.ext.reparametrisation(xs.unit_cell())
occ_Al = r.add(constraints.independent_occupancy_parameter, sc[2])
occ_K = r.add(constraints.independent_occupancy_parameter, sc[3])
occ_Na = r.add(constraints.affine_asu_occupancy_parameter,
occ_Al,
1,
occ_K,
-1,
0,
scatterer=sc[0])
occ_Ca = r.add(constraints.affine_asu_occupancy_parameter,
occ_Al,
-2,
1,
scatterer=sc[1])
r.finalise()
r.linearise()
assert approx_equal(occ_Na.value, 0.2)
assert approx_equal(occ_Ca.value, 0.3)
assert approx_equal(occ_Al.value, 0.35)
assert approx_equal(occ_K.value, 0.15)
jt0 = sparse.matrix(
2,
4,
[
{
0: 1
}, # diff occ(Al3+)
{
1: 1
}, # diff occ(K+)
{
0: 1,
1: -1
}, # diff occ(Na+)
{
0: -2
}, # diff occ(Ca2+)
])
assert sparse.approx_equal(tolerance=1e-15)(r.jacobian_transpose, jt0)
def exercise_floating_origin_dynamic_weighting(verbose=False):
from cctbx import covariance
import scitbx.math
worst_condition_number_acceptable = 10
# light elements only
xs0 = random_structure.xray_structure(elements=['C', 'C', 'C', 'O', 'N'],
use_u_aniso=True)
msg = "light elements in %s ..." % (
xs0.space_group_info().type().hall_symbol())
if verbose:
print(msg, end=' ')
fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
xs = xs0.deep_copy_scatterers()
xs.shake_adp()
xs.shake_sites_in_place(rms_difference=0.1)
for sc in xs.scatterers():
sc.flags.set_grad_site(True).set_grad_u_aniso(True)
ls = least_squares.crystallographic_ls(
fo_sq.as_xray_observations(),
constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=smtbx.utils.connectivity_table(xs)),
weighting_scheme=least_squares.unit_weighting(),
origin_fixing_restraints_type=
origin_fixing_restraints.atomic_number_weighting)
ls.build_up()
lambdas = eigensystem.real_symmetric(
ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
# assert the restrained L.S. problem is not too ill-conditionned
cond = math.log10(lambdas[0]/lambdas[-1])
if verbose:
print("normal matrix condition: %.1f" % cond)
assert cond < worst_condition_number_acceptable, msg
# one heavy element
xs0 = random_structure.xray_structure(
space_group_info=sgtbx.space_group_info('hall: P 2yb'),
elements=['Zn', 'C', 'C', 'C', 'O', 'N'],
use_u_aniso=True)
msg = "one heavy element + light elements (synthetic data) in %s ..." % (
xs0.space_group_info().type().hall_symbol())
if verbose:
print(msg, end=' ')
fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
xs = xs0.deep_copy_scatterers()
xs.shake_adp()
xs.shake_sites_in_place(rms_difference=0.1)
for sc in xs.scatterers():
sc.flags.set_grad_site(True).set_grad_u_aniso(True)
ls = least_squares.crystallographic_ls(
fo_sq.as_xray_observations(),
constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=smtbx.utils.connectivity_table(xs)),
weighting_scheme=least_squares.mainstream_shelx_weighting(),
origin_fixing_restraints_type=
origin_fixing_restraints.atomic_number_weighting)
ls.build_up()
lambdas = eigensystem.real_symmetric(
ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
# assert the restrained L.S. problem is not too ill-conditionned
cond = math.log10(lambdas[0]/lambdas[-1])
if verbose:
print("normal matrix condition: %.1f" % cond)
assert cond < worst_condition_number_acceptable, msg
# are esd's for x,y,z coordinates of the same order of magnitude?
var_cart = covariance.orthogonalize_covariance_matrix(
ls.covariance_matrix(),
xs.unit_cell(),
xs.parameter_map())
var_site_cart = covariance.extract_covariance_matrix_for_sites(
flex.size_t_range(len(xs.scatterers())),
var_cart,
xs.parameter_map())
site_esds = var_site_cart.matrix_packed_u_diagonal()
indicators = flex.double()
for i in xrange(0, len(site_esds), 3):
stats = scitbx.math.basic_statistics(site_esds[i:i+3])
indicators.append(stats.bias_corrected_standard_deviation/stats.mean)
assert indicators.all_lt(2)
# especially troublesome structure with one heavy element
# (contributed by Jonathan Coome)
xs0 = xray.structure(
crystal_symmetry=crystal.symmetry(
unit_cell=(8.4519, 8.4632, 18.7887, 90, 96.921, 90),
space_group_symbol="hall: P 2yb"),
scatterers=flex.xray_scatterer([
xray.scatterer( #0
label="ZN1",
site=(-0.736683, -0.313978, -0.246902),
u=(0.000302, 0.000323, 0.000054,
0.000011, 0.000015, -0.000004)),
xray.scatterer( #1
label="N3B",
site=(-0.721014, -0.313583, -0.134277),
u=(0.000268, 0.000237, 0.000055,
-0.000027, 0.000005, 0.000006)),
xray.scatterer( #2
label="N3A",
site=(-0.733619, -0.290423, -0.357921),
u=(0.000229, 0.000313, 0.000053,
0.000022, 0.000018, -0.000018)),
xray.scatterer( #3
label="C9B",
site=(-1.101537, -0.120157, -0.138063),
u=(0.000315, 0.000345, 0.000103,
0.000050, 0.000055, -0.000017)),
xray.scatterer( #4
label="N5B",
site=(-0.962032, -0.220345, -0.222045),
u=(0.000274, 0.000392, 0.000060,
-0.000011, -0.000001, -0.000002)),
xray.scatterer( #5
label="N1B",
site=(-0.498153, -0.402742, -0.208698),
u=(0.000252, 0.000306, 0.000063,
0.000000, 0.000007, 0.000018)),
xray.scatterer( #6
label="C3B",
site=(-0.322492, -0.472610, -0.114594),
u=(0.000302, 0.000331, 0.000085,
0.000016, -0.000013, 0.000037)),
xray.scatterer( #7
label="C4B",
site=(-0.591851, -0.368163, -0.094677),
u=(0.000262, 0.000255, 0.000073,
-0.000034, 0.000027, -0.000004)),
xray.scatterer( #8
label="N4B",
site=(-0.969383, -0.204624, -0.150014),
u=(0.000279, 0.000259, 0.000070,
-0.000009, 0.000039, 0.000000)),
xray.scatterer( #9
label="N2B",
site=(-0.470538, -0.414572, -0.135526),
u=(0.000277, 0.000282, 0.000065,
0.000003, 0.000021, -0.000006)),
xray.scatterer( #10
label="C8A",
site=(-0.679889, -0.158646, -0.385629),
u=(0.000209, 0.000290, 0.000078,
0.000060, 0.000006, 0.000016)),
xray.scatterer( #11
label="N5A",
site=(-0.649210, -0.075518, -0.263412),
u=(0.000307, 0.000335, 0.000057,
-0.000002, 0.000016, -0.000012)),
xray.scatterer( #12
label="C6B",
site=(-0.708620, -0.325965, 0.011657),
u=(0.000503, 0.000318, 0.000053,
-0.000058, 0.000032, -0.000019)),
xray.scatterer( #13
label="C10B",
site=(-1.179332, -0.083184, -0.202815),
u=(0.000280, 0.000424, 0.000136,
0.000094, 0.000006, 0.000013)),
xray.scatterer( #14
label="N1A",
site=(-0.838363, -0.532191, -0.293213),
u=(0.000312, 0.000323, 0.000060,
0.000018, 0.000011, -0.000008)),
xray.scatterer( #15
label="C3A",
site=(-0.915414, -0.671031, -0.393826),
u=(0.000319, 0.000384, 0.000078,
-0.000052, -0.000001, -0.000020)),
xray.scatterer( #16
label="C1A",
site=(-0.907466, -0.665419, -0.276011),
u=(0.000371, 0.000315, 0.000079,
0.000006, 0.000036, 0.000033)),
xray.scatterer( #17
label="C1B",
site=(-0.365085, -0.452753, -0.231927),
u=(0.000321, 0.000253, 0.000087,
-0.000024, 0.000047, -0.000034)),
xray.scatterer( #18
label="C11A",
site=(-0.598622, 0.053343, -0.227354),
u=(0.000265, 0.000409, 0.000084,
0.000088, -0.000018, -0.000030)),
xray.scatterer( #19
label="C2A",
site=(-0.958694, -0.755645, -0.337016),
u=(0.000394, 0.000350, 0.000106,
-0.000057, 0.000027, -0.000005)),
xray.scatterer( #20
label="C4A",
site=(-0.784860, -0.407601, -0.402050),
u=(0.000238, 0.000296, 0.000064,
0.000002, 0.000011, -0.000016)),
xray.scatterer( #21
label="C5A",
site=(-0.784185, -0.399716, -0.475491),
u=(0.000310, 0.000364, 0.000062,
0.000044, -0.000011, -0.000017)),
xray.scatterer( #22
label="N4A",
site=(-0.630284, -0.043981, -0.333143),
u=(0.000290, 0.000275, 0.000074,
0.000021, 0.000027, 0.000013)),
xray.scatterer( #23
label="C10A",
site=(-0.545465, 0.166922, -0.272829),
u=(0.000369, 0.000253, 0.000117,
0.000015, -0.000002, -0.000008)),
xray.scatterer( #24
label="C9A",
site=(-0.567548, 0.102272, -0.339923),
u=(0.000346, 0.000335, 0.000103,
-0.000016, 0.000037, 0.000023)),
xray.scatterer( #25
label="C11B",
site=(-1.089943, -0.146930, -0.253779),
u=(0.000262, 0.000422, 0.000102,
-0.000018, -0.000002, 0.000029)),
xray.scatterer( #26
label="N2A",
site=(-0.843385, -0.537780, -0.366515),
u=(0.000273, 0.000309, 0.000055,
-0.000012, -0.000005, -0.000018)),
xray.scatterer( #27
label="C7A",
site=(-0.674021, -0.136086, -0.457790),
u=(0.000362, 0.000378, 0.000074,
0.000043, 0.000034, 0.000016)),
xray.scatterer( #28
label="C8B",
site=(-0.843625, -0.264182, -0.102023),
u=(0.000264, 0.000275, 0.000072,
-0.000025, 0.000019, -0.000005)),
xray.scatterer( #29
label="C6A",
site=(-0.726731, -0.261702, -0.502366),
u=(0.000339, 0.000472, 0.000064,
0.000062, -0.000003, 0.000028)),
xray.scatterer( #30
label="C5B",
site=(-0.577197, -0.376753, -0.020800),
u=(0.000349, 0.000353, 0.000066,
-0.000082, -0.000022, 0.000014)),
xray.scatterer( #31
label="C2B",
site=(-0.252088, -0.497338, -0.175057),
u=(0.000251, 0.000342, 0.000119,
0.000020, 0.000034, -0.000018)),
xray.scatterer( #32
label="C7B",
site=(-0.843956, -0.268811, -0.028080),
u=(0.000344, 0.000377, 0.000078,
-0.000029, 0.000059, -0.000007)),
xray.scatterer( #33
label="F4B",
site=(-0.680814, -0.696808, -0.115056),
u=(0.000670, 0.000408, 0.000109,
-0.000099, 0.000139, -0.000031)),
xray.scatterer( #34
label="F1B",
site=(-0.780326, -0.921249, -0.073962),
u=(0.000687, 0.000357, 0.000128,
-0.000152, -0.000011, 0.000021)),
xray.scatterer( #35
label="B1B",
site=(-0.795220, -0.758128, -0.075955),
u=(0.000413, 0.000418, 0.000075,
0.000054, 0.000045, 0.000023)),
xray.scatterer( #36
label="F2B",
site=(-0.945140, -0.714626, -0.105820),
u=(0.000584, 0.001371, 0.000108,
0.000420, 0.000067, 0.000134)),
xray.scatterer( #37
label="F3B",
site=(-0.768914, -0.701660, -0.005161),
u=(0.000678, 0.000544, 0.000079,
-0.000000, 0.000090, -0.000021)),
xray.scatterer( #38
label="F1A",
site=(-0.109283, -0.252334, -0.429288),
u=(0.000427, 0.001704, 0.000125,
0.000407, 0.000041, 0.000035)),
xray.scatterer( #39
label="F4A",
site=(-0.341552, -0.262864, -0.502023),
u=(0.000640, 0.000557, 0.000081,
-0.000074, 0.000042, -0.000052)),
xray.scatterer( #40
label="F3A",
site=(-0.324533, -0.142292, -0.393215),
u=(0.000471, 0.001203, 0.000134,
0.000333, -0.000057, -0.000220)),
xray.scatterer( #41
label="F2A",
site=(-0.312838, -0.405405, -0.400231),
u=(0.002822, 0.000831, 0.000092,
-0.000648, 0.000115, 0.000027)),
xray.scatterer( #42
label="B1A",
site=(-0.271589, -0.268874, -0.430724),
u=(0.000643, 0.000443, 0.000079,
0.000040, 0.000052, -0.000034)),
xray.scatterer( #43
label="H5B",
site=(-0.475808, -0.413802, 0.004402),
u=0.005270),
xray.scatterer( #44
label="H6B",
site=(-0.699519, -0.326233, 0.062781),
u=0.019940),
xray.scatterer( #45
label="H3B",
site=(-0.283410, -0.484757, -0.063922),
u=0.029990),
xray.scatterer( #46
label="H1B",
site=(-0.357103, -0.451819, -0.284911),
u=0.031070),
xray.scatterer( #47
label="H10A",
site=(-0.495517, 0.268296, -0.256187),
u=0.027610),
xray.scatterer( #48
label="H2B",
site=(-0.147129, -0.535141, -0.174699),
u=0.017930),
xray.scatterer( #49
label="H7A",
site=(-0.643658, -0.031387, -0.475357),
u=0.020200),
xray.scatterer( #50
label="H1A",
site=(-0.912757, -0.691043, -0.227554),
u=0.033320),
xray.scatterer( #51
label="H7B",
site=(-0.933670, -0.241189, -0.010263),
u=0.021310),
xray.scatterer( #52
label="H11B",
site=(-1.107736, -0.155470, -0.311996),
u=0.041500),
xray.scatterer( #53
label="H9A",
site=(-0.539908, 0.139753, -0.382281),
u=0.007130),
xray.scatterer( #54
label="H10B",
site=(-1.265944, -0.029610, -0.212398),
u=0.030910),
xray.scatterer( #55
label="H3A",
site=(-0.934728, -0.691149, -0.450551),
u=0.038950),
xray.scatterer( #56
label="H5A",
site=(-0.833654, -0.487479, -0.508239),
u=0.031150),
xray.scatterer( #57
label="H6A",
site=(-0.742871, -0.242269, -0.558157),
u=0.050490),
xray.scatterer( #58
label="H9B",
site=(-1.120150, -0.093752, -0.090706),
u=0.039310),
xray.scatterer( #59
label="H11A",
site=(-0.593074, 0.054973, -0.180370),
u=0.055810),
xray.scatterer( #60
label="H2A",
site=(-0.999576, -0.842158, -0.340837),
u=0.057030)
]))
fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
for hydrogen_flag in (True, False):
xs = xs0.deep_copy_scatterers()
if not hydrogen_flag:
xs.select_inplace(~xs.element_selection('H'))
xs.shake_adp()
xs.shake_sites_in_place(rms_difference=0.1)
for sc in xs.scatterers():
sc.flags.set_grad_site(True).set_grad_u_aniso(False)
ls = least_squares.crystallographic_ls(
fo_sq.as_xray_observations(),
constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=smtbx.utils.connectivity_table(xs)),
weighting_scheme=least_squares.unit_weighting(),
origin_fixing_restraints_type=
origin_fixing_restraints.atomic_number_weighting)
ls.build_up()
lambdas = eigensystem.real_symmetric(
ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
# assert the restrained L.S. problem is not too ill-conditionned
cond = math.log10(lambdas[0]/lambdas[-1])
msg = ("one heavy element + light elements (real data) %s Hydrogens: %.1f"
% (['without', 'with'][hydrogen_flag], cond))
if verbose: print(msg)
assert cond < worst_condition_number_acceptable, msg
# are esd's for x,y,z coordinates of the same order of magnitude?
var_cart = covariance.orthogonalize_covariance_matrix(
ls.covariance_matrix(),
xs.unit_cell(),
xs.parameter_map())
var_site_cart = covariance.extract_covariance_matrix_for_sites(
flex.size_t_range(len(xs.scatterers())),
var_cart,
xs.parameter_map())
site_esds = var_site_cart.matrix_packed_u_diagonal()
indicators = flex.double()
for i in xrange(0, len(site_esds), 3):
stats = scitbx.math.basic_statistics(site_esds[i:i+3])
indicators.append(stats.bias_corrected_standard_deviation/stats.mean)
assert indicators.all_lt(1)
def exercise_hl_ab_only(anomalous_flag):
cs = crystal.symmetry(unit_cell=(3.95738, 5.1446, 6.72755, 83, 109, 129),
space_group_symbol="P1")
if (not anomalous_flag):
i = [(-1, 0, 1), (-1, 1, 1), (0, -1, 1), (0, 0, 1), (0, 0, 2),
(0, 1, 0), (0, 1, 1), (1, -1, 0)]
else:
i = [(-1, 0, 1), (1, 0, -1), (-1, 1, 1), (1, -1, -1), (0, -1, 1),
(0, 1, -1), (0, 0, 1), (0, 0, -1), (0, 0, 2), (0, 0, -2),
(0, 1, 0), (0, -1, 0), (0, 1, 1), (0, -1, -1), (1, -1, 0),
(-1, 1, 0)]
ms = miller.set(crystal_symmetry=cs,
indices=flex.miller_index(i),
anomalous_flag=anomalous_flag)
ma = ms.array(data=flex.size_t_range(ms.indices().size()).as_double() + 1)
mtz_dataset = ma.as_mtz_dataset(column_root_label="HA")
columns = mtz_dataset.columns()
if (not anomalous_flag):
assert columns.size() == 4
c = columns[3]
assert c.label() == "HA"
c.set_type("A")
values = c.extract_values()
selection_valid = c.selection_valid()
c = mtz_dataset.add_column(label="HB", type="A")
c.set_values(values=-values, selection_valid=selection_valid)
else:
assert columns.size() == 5
for i, l in [(3, "HA(+)"), (4, "HA(-)")]:
c = columns[i]
assert c.label() == l
if (i == 4):
c.set_label("HB(+)")
c.set_type("A")
for i, l in [(3, "HA(-)"), (4, "HB(-)")]:
c = columns[i]
values = c.extract_values()
selection_valid = c.selection_valid()
c = mtz_dataset.add_column(label=l, type="A")
c.set_values(values=-values, selection_valid=selection_valid)
mtz_obj = mtz_dataset.mtz_object()
mas = mtz_obj.as_miller_arrays()
assert len(mas) == 1
assert approx_equal(mas[0].indices(), ma.indices())
if (not anomalous_flag):
assert approx_equal(mas[0].data(), [(1, -1, 0, 0), (2, -2, 0, 0),
(3, -3, 0, 0), (4, -4, 0, 0),
(5, -5, 0, 0), (6, -6, 0, 0),
(7, -7, 0, 0), (8, -8, 0, 0)])
else:
assert approx_equal(mas[0].data(), [(1, 2, 0, 0), (-1, -2, 0, 0),
(3, 4, 0, 0), (-3, -4, 0, 0),
(5, 6, 0, 0), (-5, -6, 0, 0),
(7, 8, 0, 0), (-7, -8, 0, 0),
(9, 10, 0, 0), (-9, -10, 0, 0),
(11, 12, 0, 0), (-11, -12, 0, 0),
(13, 14, 0, 0), (-13, -14, 0, 0),
(15, 16, 0, 0), (-15, -16, 0, 0)])
#
columns = mtz_dataset.columns()
columns[-1].set_type("F")
try:
mtz_obj.as_miller_arrays()
except RuntimeError as e:
if (not anomalous_flag):
assert str(e) == 'Invalid MTZ column combination' \
' (incomplete Hendrickson-Lattman array),' \
' column labels: "HA", "HB" column types: "A", "F"'
else:
assert str(e) == 'Invalid MTZ column combination' \
' (incomplete Hendrickson-Lattman array),' \
' column labels: "HA(-)", "HB(-)" column types: "A", "F"'
else:
raise Exception_expected
def exercise_affine_occupancy_parameter():
xs = xray.structure(
crystal_symmetry=crystal.symmetry(unit_cell=(), space_group_symbol='hall: P 1'),
scatterers=flex.xray_scatterer((
xray.scatterer('C0', occupancy=1),
xray.scatterer('C1', occupancy=1),
xray.scatterer('C2', occupancy=1),
xray.scatterer('C3', occupancy=1),
)))
sc = xs.scatterers()
sc.flags_set_grad_occupancy(flex.size_t_range(4))
# Two occupancies adding up to 1 (most common case of disorder)
r = constraints.ext.reparametrisation(xs.unit_cell())
occ_1 = r.add(constraints.independent_occupancy_parameter, sc[1])
occ_3 = r.add(constraints.affine_asu_occupancy_parameter,
dependee=occ_1, a=-1, b=1, scatterer=sc[3])
r.finalise()
r.linearise()
assert approx_equal(occ_1.value, 1)
assert approx_equal(occ_3.value, 0)
jt0 = sparse.matrix(1, 2,
[ {0:1}, # 1st col = derivatives of occ_1
{0:-1}, # 2nd col = derivatives of occ_3
])
assert sparse.approx_equal(tolerance=1e-15)(r.jacobian_transpose, jt0)
# Example illustrating the instruction SUMP in SHELX 97 manual (p. 7-26)
# We disregard the issue of the special position which is orthogonal to the
# point we want to test here.
xs = xray.structure(
crystal_symmetry=crystal.symmetry(unit_cell=(), space_group_symbol='hall: P 1'),
scatterers=flex.xray_scatterer((
xray.scatterer('Na+', occupancy=1),
xray.scatterer('Ca2+', occupancy=1),
xray.scatterer('Al3+', occupancy=0.35),
xray.scatterer('K+', occupancy=0.15),
)))
sc = xs.scatterers()
sc.flags_set_grad_occupancy(flex.size_t_range(4))
# The constraints are:
# fully occupied: occ(Na+) + occ(Ca2+) + occ(Al3+) + occ(K+) = 1
# average charge +2: occ(Na+) + 2 occ(Ca2+) + 3 occ(Al3+) + occ(K+) = +2
# This can be solved as:
# occ(Na+) = occ(Al3+) - occ(K+)
# occ(Ca2+) = 1 - 2 occ(Al3+)
r = constraints.ext.reparametrisation(xs.unit_cell())
occ_Al = r.add(constraints.independent_occupancy_parameter, sc[2])
occ_K = r.add(constraints.independent_occupancy_parameter, sc[3])
occ_Na = r.add(constraints.affine_asu_occupancy_parameter,
occ_Al, 1, occ_K, -1, 0, scatterer=sc[0])
occ_Ca = r.add(constraints.affine_asu_occupancy_parameter,
occ_Al, -2, 1, scatterer=sc[1])
r.finalise()
r.linearise()
assert approx_equal(occ_Na.value, 0.2)
assert approx_equal(occ_Ca.value, 0.3)
assert approx_equal(occ_Al.value, 0.35)
assert approx_equal(occ_K.value, 0.15)
jt0 = sparse.matrix(2, 4,
[
{0:1}, # diff occ(Al3+)
{1:1} , # diff occ(K+)
{0:1, 1:-1}, # diff occ(Na+)
{0:-2}, # diff occ(Ca2+)
])
assert sparse.approx_equal(tolerance=1e-15)(r.jacobian_transpose, jt0)
def exercise_hl_ab_only(anomalous_flag):
cs = crystal.symmetry(unit_cell=(3.95738, 5.1446, 6.72755, 83, 109, 129), space_group_symbol="P1")
if not anomalous_flag:
i = [(-1, 0, 1), (-1, 1, 1), (0, -1, 1), (0, 0, 1), (0, 0, 2), (0, 1, 0), (0, 1, 1), (1, -1, 0)]
else:
i = [
(-1, 0, 1),
(1, 0, -1),
(-1, 1, 1),
(1, -1, -1),
(0, -1, 1),
(0, 1, -1),
(0, 0, 1),
(0, 0, -1),
(0, 0, 2),
(0, 0, -2),
(0, 1, 0),
(0, -1, 0),
(0, 1, 1),
(0, -1, -1),
(1, -1, 0),
(-1, 1, 0),
]
ms = miller.set(crystal_symmetry=cs, indices=flex.miller_index(i), anomalous_flag=anomalous_flag)
ma = ms.array(data=flex.size_t_range(ms.indices().size()).as_double() + 1)
mtz_dataset = ma.as_mtz_dataset(column_root_label="HA")
columns = mtz_dataset.columns()
if not anomalous_flag:
assert columns.size() == 4
c = columns[3]
assert c.label() == "HA"
c.set_type("A")
values = c.extract_values()
selection_valid = c.selection_valid()
c = mtz_dataset.add_column(label="HB", type="A")
c.set_values(values=-values, selection_valid=selection_valid)
else:
assert columns.size() == 5
for i, l in [(3, "HA(+)"), (4, "HA(-)")]:
c = columns[i]
assert c.label() == l
if i == 4:
c.set_label("HB(+)")
c.set_type("A")
for i, l in [(3, "HA(-)"), (4, "HB(-)")]:
c = columns[i]
values = c.extract_values()
selection_valid = c.selection_valid()
c = mtz_dataset.add_column(label=l, type="A")
c.set_values(values=-values, selection_valid=selection_valid)
mtz_obj = mtz_dataset.mtz_object()
mas = mtz_obj.as_miller_arrays()
assert len(mas) == 1
assert approx_equal(mas[0].indices(), ma.indices())
if not anomalous_flag:
assert approx_equal(
mas[0].data(),
[
(1, -1, 0, 0),
(2, -2, 0, 0),
(3, -3, 0, 0),
(4, -4, 0, 0),
(5, -5, 0, 0),
(6, -6, 0, 0),
(7, -7, 0, 0),
(8, -8, 0, 0),
],
)
else:
assert approx_equal(
mas[0].data(),
[
(1, 2, 0, 0),
(-1, -2, 0, 0),
(3, 4, 0, 0),
(-3, -4, 0, 0),
(5, 6, 0, 0),
(-5, -6, 0, 0),
(7, 8, 0, 0),
(-7, -8, 0, 0),
(9, 10, 0, 0),
(-9, -10, 0, 0),
(11, 12, 0, 0),
(-11, -12, 0, 0),
(13, 14, 0, 0),
(-13, -14, 0, 0),
(15, 16, 0, 0),
(-15, -16, 0, 0),
],
)
#
columns = mtz_dataset.columns()
columns[-1].set_type("F")
try:
mtz_obj.as_miller_arrays()
except RuntimeError, e:
if not anomalous_flag:
assert (
str(e) == "Invalid MTZ column combination"
" (incomplete Hendrickson-Lattman array),"
' column labels: "HA", "HB" column types: "A", "F"'
)
else:
assert (
str(e) == "Invalid MTZ column combination"
" (incomplete Hendrickson-Lattman array),"
' column labels: "HA(-)", "HB(-)" column types: "A", "F"'
)
def optimise_parameters(self, fo_sq, fc_sq,
scale_factor, n_independent_params):
""" Find optimal values of a and b that give a flat analysis of the variance
when binned by Fc/max(Fc), and a goodness of fit close to 1.
This is done in a grid search fashion similar to Shelxl.
self is not modified in place; instead a new instance of the weighting
scheme is returned.
It is intended that f_calc should already contain the contribution from
f_mask (if a solvent mask is used).
"""
assert fc_sq.is_xray_intensity_array()
weighting = ext.mainstream_shelx_weighting(a=self.a, b=self.b)
def compute_chi_sq(fo_sq, fc_sq, a,b):
weighting.a = a
weighting.b = b
weights = weighting(
fo_sq.data(), fo_sq.sigmas(), fc_sq.data(), scale_factor)
return (flex.sum(
weights * flex.pow2(fo_sq.data() - scale_factor * fc_sq.data())))
fo_sq = fo_sq.deep_copy()
fo_sq.data().set_selected(fo_sq.data() < 0, 0)
fo2 = fo_sq.data().deep_copy()
fo2 /= scale_factor
sigmas = fo_sq.sigmas() / scale_factor
sigmas_sq = flex.pow2(sigmas)
fc2 = fc_sq.data()
# determine starting values for a and b, formulae taken from shelxl code
p = (fo2 + 2 * fc2)/3
p_sq = flex.pow2(p)
x = flex.sum((flex.pow2(fo2-fc2)-sigmas) * (p_sq/sigmas_sq))
y = flex.sum( flex.pow2(p_sq)/sigmas_sq)
z = flex.sum(p)
start_a = math.sqrt(max(0.0001, 0.64*x/max(1e-8, y)))
start_b = 0.5 * z * start_a**2 /fo_sq.size()
a_step = 0.2 * start_a
b_step = 0.4 * start_b
# sort data and setup binning by fc/fc_max
fc_sq_over_fc_sq_max = fc_sq.data()/flex.max(fc_sq.data())
permutation = flex.sort_permutation(fc_sq_over_fc_sq_max)
fc_sq_over_fc_sq_max = fc_sq.customized_copy(
data=fc_sq_over_fc_sq_max).select(permutation)
fc_sq = fc_sq.select(permutation)
fo_sq = fo_sq.select(permutation)
n_bins = 10
bin_max = 0
bin_limits = flex.size_t(1, 0)
bin_count = flex.size_t()
for i in range(n_bins):
bin_limits.append(int(math.ceil((i+1) * fc_sq.size()/n_bins)))
bin_count.append(bin_limits[i+1] - bin_limits[i])
n = fo_sq.size()//(fo_sq.size()-n_independent_params)
# search on a 9x9 grid to determine best values of a and b
gridding = flex.grid(9,9)
while (a_step > 1e-4 and b_step > 5e-3):
tmp = flex.double(gridding, 0)
binned_chi_sq = [tmp.deep_copy() for i in range(n_bins)]
start_a = max(start_a, 4*a_step) - 4*a_step
start_b = max(start_b, 4*b_step) - 4*b_step
for i_bin in range(n_bins):
sel = flex.size_t_range(bin_limits[i_bin], bin_limits[i_bin+1])
fc2 = fc_sq.select(sel)
fo2 = fo_sq.select(sel)
b = start_b
for j in range(9):
a = start_a
b += b_step
for k in range(9):
a += a_step
binned_chi_sq[i_bin][j,k] += compute_chi_sq(fo2, fc2, a, b)
min_variance = 9e9
j_min, k_min = (0, 0)
for j in range(9):
for k in range(9):
variance = 0
for i_bin in range(n_bins):
if bin_count[i_bin] == 0: continue
goof = math.sqrt(binned_chi_sq[i_bin][j,k]*n/bin_count[i_bin])
variance += (goof-1)**2
min_variance = min(variance, min_variance)
if variance == min_variance:
j_min = j
k_min = k
start_a += k_min*a_step
start_b += j_min*b_step
if k_min == 8:
a_step *= 2
continue
elif k_min != 0:
a_step /= 4
if j_min == 8:
b_step *= 2
continue
elif j_min != 0:
b_step /=4
if start_a <= 1e-4: a_step /= 4
if start_b <= 1e-3: b_step /= 4
if start_a > 0.2:
start_a = 0.2
start_b = 0
weighting.a = start_a
weighting.b = start_b
return weighting
def exercise_floating_origin_dynamic_weighting(verbose=False):
from cctbx import covariance
import scitbx.math
worst_condition_number_acceptable = 10
# light elements only
xs0 = random_structure.xray_structure(elements=['C', 'C', 'C', 'O', 'N'],
use_u_aniso=True)
fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
xs = xs0.deep_copy_scatterers()
xs.shake_adp()
xs.shake_sites_in_place(rms_difference=0.1)
for sc in xs.scatterers():
sc.flags.set_grad_site(True).set_grad_u_aniso(True)
ls = least_squares.crystallographic_ls(
fo_sq.as_xray_observations(),
constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=smtbx.utils.connectivity_table(xs)),
weighting_scheme=least_squares.unit_weighting(),
origin_fixing_restraints_type=
origin_fixing_restraints.atomic_number_weighting)
ls.build_up()
lambdas = eigensystem.real_symmetric(
ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
# assert the restrained L.S. problem is not too ill-conditionned
cond = math.log10(lambdas[0]/lambdas[-1])
msg = "light elements: %.1f" % cond
if verbose: print msg
assert cond < worst_condition_number_acceptable, msg
# one heavy element
xs0 = random_structure.xray_structure(
space_group_info=sgtbx.space_group_info('hall: P 2yb'),
elements=['Zn', 'C', 'C', 'C', 'O', 'N'],
use_u_aniso=True)
fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
xs = xs0.deep_copy_scatterers()
xs.shake_adp()
xs.shake_sites_in_place(rms_difference=0.1)
for sc in xs.scatterers():
sc.flags.set_grad_site(True).set_grad_u_aniso(True)
ls = least_squares.crystallographic_ls(
fo_sq.as_xray_observations(),
constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=smtbx.utils.connectivity_table(xs)),
weighting_scheme=least_squares.mainstream_shelx_weighting(),
origin_fixing_restraints_type=
origin_fixing_restraints.atomic_number_weighting)
ls.build_up()
lambdas = eigensystem.real_symmetric(
ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
# assert the restrained L.S. problem is not too ill-conditionned
cond = math.log10(lambdas[0]/lambdas[-1])
msg = "one heavy element + light elements (synthetic data): %.1f" % cond
if verbose: print msg
assert cond < worst_condition_number_acceptable, msg
# are esd's for x,y,z coordinates of the same order of magnitude?
var_cart = covariance.orthogonalize_covariance_matrix(
ls.covariance_matrix(),
xs.unit_cell(),
xs.parameter_map())
var_site_cart = covariance.extract_covariance_matrix_for_sites(
flex.size_t_range(len(xs.scatterers())),
var_cart,
xs.parameter_map())
site_esds = var_site_cart.matrix_packed_u_diagonal()
indicators = flex.double()
for i in xrange(0, len(site_esds), 3):
stats = scitbx.math.basic_statistics(site_esds[i:i+3])
indicators.append(stats.bias_corrected_standard_deviation/stats.mean)
assert indicators.all_lt(1)
# especially troublesome structure with one heavy element
# (contributed by Jonathan Coome)
xs0 = xray.structure(
crystal_symmetry=crystal.symmetry(
unit_cell=(8.4519, 8.4632, 18.7887, 90, 96.921, 90),
space_group_symbol="hall: P 2yb"),
scatterers=flex.xray_scatterer([
xray.scatterer( #0
label="ZN1",
site=(-0.736683, -0.313978, -0.246902),
u=(0.000302, 0.000323, 0.000054,
0.000011, 0.000015, -0.000004)),
xray.scatterer( #1
label="N3B",
site=(-0.721014, -0.313583, -0.134277),
u=(0.000268, 0.000237, 0.000055,
-0.000027, 0.000005, 0.000006)),
xray.scatterer( #2
label="N3A",
site=(-0.733619, -0.290423, -0.357921),
u=(0.000229, 0.000313, 0.000053,
0.000022, 0.000018, -0.000018)),
xray.scatterer( #3
label="C9B",
site=(-1.101537, -0.120157, -0.138063),
u=(0.000315, 0.000345, 0.000103,
0.000050, 0.000055, -0.000017)),
xray.scatterer( #4
label="N5B",
site=(-0.962032, -0.220345, -0.222045),
u=(0.000274, 0.000392, 0.000060,
-0.000011, -0.000001, -0.000002)),
xray.scatterer( #5
label="N1B",
site=(-0.498153, -0.402742, -0.208698),
u=(0.000252, 0.000306, 0.000063,
0.000000, 0.000007, 0.000018)),
xray.scatterer( #6
label="C3B",
site=(-0.322492, -0.472610, -0.114594),
u=(0.000302, 0.000331, 0.000085,
0.000016, -0.000013, 0.000037)),
xray.scatterer( #7
label="C4B",
site=(-0.591851, -0.368163, -0.094677),
u=(0.000262, 0.000255, 0.000073,
-0.000034, 0.000027, -0.000004)),
xray.scatterer( #8
label="N4B",
site=(-0.969383, -0.204624, -0.150014),
u=(0.000279, 0.000259, 0.000070,
-0.000009, 0.000039, 0.000000)),
xray.scatterer( #9
label="N2B",
site=(-0.470538, -0.414572, -0.135526),
u=(0.000277, 0.000282, 0.000065,
0.000003, 0.000021, -0.000006)),
xray.scatterer( #10
label="C8A",
site=(-0.679889, -0.158646, -0.385629),
u=(0.000209, 0.000290, 0.000078,
0.000060, 0.000006, 0.000016)),
xray.scatterer( #11
label="N5A",
site=(-0.649210, -0.075518, -0.263412),
u=(0.000307, 0.000335, 0.000057,
-0.000002, 0.000016, -0.000012)),
xray.scatterer( #12
label="C6B",
site=(-0.708620, -0.325965, 0.011657),
u=(0.000503, 0.000318, 0.000053,
-0.000058, 0.000032, -0.000019)),
xray.scatterer( #13
label="C10B",
site=(-1.179332, -0.083184, -0.202815),
u=(0.000280, 0.000424, 0.000136,
0.000094, 0.000006, 0.000013)),
xray.scatterer( #14
label="N1A",
site=(-0.838363, -0.532191, -0.293213),
u=(0.000312, 0.000323, 0.000060,
0.000018, 0.000011, -0.000008)),
xray.scatterer( #15
label="C3A",
site=(-0.915414, -0.671031, -0.393826),
u=(0.000319, 0.000384, 0.000078,
-0.000052, -0.000001, -0.000020)),
xray.scatterer( #16
label="C1A",
site=(-0.907466, -0.665419, -0.276011),
u=(0.000371, 0.000315, 0.000079,
0.000006, 0.000036, 0.000033)),
xray.scatterer( #17
label="C1B",
site=(-0.365085, -0.452753, -0.231927),
u=(0.000321, 0.000253, 0.000087,
-0.000024, 0.000047, -0.000034)),
xray.scatterer( #18
label="C11A",
site=(-0.598622, 0.053343, -0.227354),
u=(0.000265, 0.000409, 0.000084,
0.000088, -0.000018, -0.000030)),
xray.scatterer( #19
label="C2A",
site=(-0.958694, -0.755645, -0.337016),
u=(0.000394, 0.000350, 0.000106,
-0.000057, 0.000027, -0.000005)),
xray.scatterer( #20
label="C4A",
site=(-0.784860, -0.407601, -0.402050),
u=(0.000238, 0.000296, 0.000064,
0.000002, 0.000011, -0.000016)),
xray.scatterer( #21
label="C5A",
site=(-0.784185, -0.399716, -0.475491),
u=(0.000310, 0.000364, 0.000062,
0.000044, -0.000011, -0.000017)),
xray.scatterer( #22
label="N4A",
site=(-0.630284, -0.043981, -0.333143),
u=(0.000290, 0.000275, 0.000074,
0.000021, 0.000027, 0.000013)),
xray.scatterer( #23
label="C10A",
site=(-0.545465, 0.166922, -0.272829),
u=(0.000369, 0.000253, 0.000117,
0.000015, -0.000002, -0.000008)),
xray.scatterer( #24
label="C9A",
site=(-0.567548, 0.102272, -0.339923),
u=(0.000346, 0.000335, 0.000103,
-0.000016, 0.000037, 0.000023)),
xray.scatterer( #25
label="C11B",
site=(-1.089943, -0.146930, -0.253779),
u=(0.000262, 0.000422, 0.000102,
-0.000018, -0.000002, 0.000029)),
xray.scatterer( #26
label="N2A",
site=(-0.843385, -0.537780, -0.366515),
u=(0.000273, 0.000309, 0.000055,
-0.000012, -0.000005, -0.000018)),
xray.scatterer( #27
label="C7A",
site=(-0.674021, -0.136086, -0.457790),
u=(0.000362, 0.000378, 0.000074,
0.000043, 0.000034, 0.000016)),
xray.scatterer( #28
label="C8B",
site=(-0.843625, -0.264182, -0.102023),
u=(0.000264, 0.000275, 0.000072,
-0.000025, 0.000019, -0.000005)),
xray.scatterer( #29
label="C6A",
site=(-0.726731, -0.261702, -0.502366),
u=(0.000339, 0.000472, 0.000064,
0.000062, -0.000003, 0.000028)),
xray.scatterer( #30
label="C5B",
site=(-0.577197, -0.376753, -0.020800),
u=(0.000349, 0.000353, 0.000066,
-0.000082, -0.000022, 0.000014)),
xray.scatterer( #31
label="C2B",
site=(-0.252088, -0.497338, -0.175057),
u=(0.000251, 0.000342, 0.000119,
0.000020, 0.000034, -0.000018)),
xray.scatterer( #32
label="C7B",
site=(-0.843956, -0.268811, -0.028080),
u=(0.000344, 0.000377, 0.000078,
-0.000029, 0.000059, -0.000007)),
xray.scatterer( #33
label="F4B",
site=(-0.680814, -0.696808, -0.115056),
u=(0.000670, 0.000408, 0.000109,
-0.000099, 0.000139, -0.000031)),
xray.scatterer( #34
label="F1B",
site=(-0.780326, -0.921249, -0.073962),
u=(0.000687, 0.000357, 0.000128,
-0.000152, -0.000011, 0.000021)),
xray.scatterer( #35
label="B1B",
site=(-0.795220, -0.758128, -0.075955),
u=(0.000413, 0.000418, 0.000075,
0.000054, 0.000045, 0.000023)),
xray.scatterer( #36
label="F2B",
site=(-0.945140, -0.714626, -0.105820),
u=(0.000584, 0.001371, 0.000108,
0.000420, 0.000067, 0.000134)),
xray.scatterer( #37
label="F3B",
site=(-0.768914, -0.701660, -0.005161),
u=(0.000678, 0.000544, 0.000079,
-0.000000, 0.000090, -0.000021)),
xray.scatterer( #38
label="F1A",
site=(-0.109283, -0.252334, -0.429288),
u=(0.000427, 0.001704, 0.000125,
0.000407, 0.000041, 0.000035)),
xray.scatterer( #39
label="F4A",
site=(-0.341552, -0.262864, -0.502023),
u=(0.000640, 0.000557, 0.000081,
-0.000074, 0.000042, -0.000052)),
xray.scatterer( #40
label="F3A",
site=(-0.324533, -0.142292, -0.393215),
u=(0.000471, 0.001203, 0.000134,
0.000333, -0.000057, -0.000220)),
xray.scatterer( #41
label="F2A",
site=(-0.312838, -0.405405, -0.400231),
u=(0.002822, 0.000831, 0.000092,
-0.000648, 0.000115, 0.000027)),
xray.scatterer( #42
label="B1A",
site=(-0.271589, -0.268874, -0.430724),
u=(0.000643, 0.000443, 0.000079,
0.000040, 0.000052, -0.000034)),
xray.scatterer( #43
label="H5B",
site=(-0.475808, -0.413802, 0.004402),
u=0.005270),
xray.scatterer( #44
label="H6B",
site=(-0.699519, -0.326233, 0.062781),
u=0.019940),
xray.scatterer( #45
label="H3B",
site=(-0.283410, -0.484757, -0.063922),
u=0.029990),
xray.scatterer( #46
label="H1B",
site=(-0.357103, -0.451819, -0.284911),
u=0.031070),
xray.scatterer( #47
label="H10A",
site=(-0.495517, 0.268296, -0.256187),
u=0.027610),
xray.scatterer( #48
label="H2B",
site=(-0.147129, -0.535141, -0.174699),
u=0.017930),
xray.scatterer( #49
label="H7A",
site=(-0.643658, -0.031387, -0.475357),
u=0.020200),
xray.scatterer( #50
label="H1A",
site=(-0.912757, -0.691043, -0.227554),
u=0.033320),
xray.scatterer( #51
label="H7B",
site=(-0.933670, -0.241189, -0.010263),
u=0.021310),
xray.scatterer( #52
label="H11B",
site=(-1.107736, -0.155470, -0.311996),
u=0.041500),
xray.scatterer( #53
label="H9A",
site=(-0.539908, 0.139753, -0.382281),
u=0.007130),
xray.scatterer( #54
label="H10B",
site=(-1.265944, -0.029610, -0.212398),
u=0.030910),
xray.scatterer( #55
label="H3A",
site=(-0.934728, -0.691149, -0.450551),
u=0.038950),
xray.scatterer( #56
label="H5A",
site=(-0.833654, -0.487479, -0.508239),
u=0.031150),
xray.scatterer( #57
label="H6A",
site=(-0.742871, -0.242269, -0.558157),
u=0.050490),
xray.scatterer( #58
label="H9B",
site=(-1.120150, -0.093752, -0.090706),
u=0.039310),
xray.scatterer( #59
label="H11A",
site=(-0.593074, 0.054973, -0.180370),
u=0.055810),
xray.scatterer( #60
label="H2A",
site=(-0.999576, -0.842158, -0.340837),
u=0.057030)
]))
fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
for hydrogen_flag in (True, False):
xs = xs0.deep_copy_scatterers()
if not hydrogen_flag:
xs.select_inplace(~xs.element_selection('H'))
xs.shake_adp()
xs.shake_sites_in_place(rms_difference=0.1)
for sc in xs.scatterers():
sc.flags.set_grad_site(True).set_grad_u_aniso(False)
ls = least_squares.crystallographic_ls(
fo_sq.as_xray_observations(),
constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=smtbx.utils.connectivity_table(xs)),
weighting_scheme=least_squares.unit_weighting(),
origin_fixing_restraints_type=
origin_fixing_restraints.atomic_number_weighting)
ls.build_up()
lambdas = eigensystem.real_symmetric(
ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
# assert the restrained L.S. problem is not too ill-conditionned
cond = math.log10(lambdas[0]/lambdas[-1])
msg = ("one heavy element + light elements (real data) %s Hydrogens: %.1f"
% (['without', 'with'][hydrogen_flag], cond))
if verbose: print msg
assert cond < worst_condition_number_acceptable, msg
# are esd's for x,y,z coordinates of the same order of magnitude?
var_cart = covariance.orthogonalize_covariance_matrix(
ls.covariance_matrix(),
xs.unit_cell(),
xs.parameter_map())
var_site_cart = covariance.extract_covariance_matrix_for_sites(
flex.size_t_range(len(xs.scatterers())),
var_cart,
xs.parameter_map())
site_esds = var_site_cart.matrix_packed_u_diagonal()
indicators = flex.double()
for i in xrange(0, len(site_esds), 3):
stats = scitbx.math.basic_statistics(site_esds[i:i+3])
indicators.append(stats.bias_corrected_standard_deviation/stats.mean)
assert indicators.all_lt(1)
def exercise(verbose=0):
distance_ideal = 1.8
default_vdw_distance = 3.6
vdw_1_4_factor = 3.5/3.6
sites_cart_manual = flex.vec3_double([
(1,3,0), (2,3,0), (3,2,0), (3,1,0), (4,1,0), (3,4,0), (4,3,0), (5,3,0),
(6,2,0), (7,2,0), (8,3,0), (7,4,0), (6,4,0), (7,5,0), (6,6,0), (8,6,0)])
bond_proxies = geometry_restraints.bond_sorted_asu_proxies(asu_mappings=None)
for i_seqs in [(0,1),(1,2),(2,3),(3,4),(1,5),(2,6),(5,6),
(6,7),(7,8),(8,9),(9,10),(10,11),(11,12),
(12,7),(11,13),(13,14),(14,15),(15,13)]:
bond_proxies.process(geometry_restraints.bond_simple_proxy(
i_seqs=i_seqs, distance_ideal=distance_ideal, weight=100))
angle_proxies = geometry_restraints.shared_angle_proxy()
for i_seqs,angle_ideal in [[(0,1,2),135],
[(0,1,5),135],
[(1,2,3),135],
[(3,2,6),135],
[(2,3,4),120],
[(1,2,6),90],
[(2,6,5),90],
[(6,5,1),90],
[(5,1,2),90],
[(2,6,7),135],
[(5,6,7),135],
[(6,7,8),120],
[(6,7,12),120],
[(7,8,9),120],
[(8,9,10),120],
[(9,10,11),120],
[(10,11,12),120],
[(11,12,7),120],
[(12,7,8),120],
[(10,11,13),120],
[(12,11,13),120],
[(11,13,15),150],
[(11,13,14),150],
[(13,15,14),60],
[(15,14,13),60],
[(14,13,15),60]]:
angle_proxies.append(geometry_restraints.angle_proxy(
i_seqs=i_seqs, angle_ideal=angle_ideal, weight=1))
if (0 or verbose):
dump_pdb(file_name="manual.pdb", sites_cart=sites_cart_manual)
for traditional_convergence_test in [True,False]:
for sites_cart_selection in [True, False]:
sites_cart = sites_cart_manual.deep_copy()
if sites_cart_selection:
sites_cart_selection = flex.bool(sites_cart.size(), True)
sites_cart_selection[1] = False
assert bond_proxies.asu.size() == 0
bond_params_table = geometry_restraints.extract_bond_params(
n_seq=sites_cart.size(),
bond_simple_proxies=bond_proxies.simple)
manager = geometry_restraints.manager.manager(
bond_params_table=bond_params_table,
angle_proxies=angle_proxies)
minimized = geometry_restraints.lbfgs.lbfgs(
sites_cart=sites_cart,
geometry_restraints_manager=manager,
lbfgs_termination_params=scitbx.lbfgs.termination_parameters(
traditional_convergence_test=traditional_convergence_test,
drop_convergence_test_max_drop_eps=1.e-20,
drop_convergence_test_iteration_coefficient=1,
max_iterations=1000),
sites_cart_selection=sites_cart_selection,
)
assert minimized.minimizer.iter() > 100
sites_cart_minimized_1 = sites_cart.deep_copy()
if (0 or verbose):
dump_pdb(
file_name="minimized_1.pdb", sites_cart=sites_cart_minimized_1)
bond_deltas = geometry_restraints.bond_deltas(
sites_cart=sites_cart_minimized_1,
proxies=bond_proxies.simple)
angle_deltas = geometry_restraints.angle_deltas(
sites_cart=sites_cart_minimized_1,
proxies=angle_proxies)
if (0 or verbose):
for proxy,delta in zip(bond_proxies.simple, bond_deltas):
print "bond:", proxy.i_seqs, delta
for proxy,delta in zip(angle_proxies, angle_deltas):
print "angle:", proxy.i_seqs, delta
assert is_below_limit(
value=flex.max(flex.abs(bond_deltas)), limit=0, eps=1.e-6)
assert is_below_limit(
value=flex.max(flex.abs(angle_deltas)), limit=0, eps=2.e-6)
sites_cart += matrix.col((1,1,0)) - matrix.col(sites_cart.min())
unit_cell_lengths = list( matrix.col(sites_cart.max())
+ matrix.col((1,-1.2,4)))
unit_cell_lengths[1] *= 2
unit_cell_lengths[2] *= 2
xray_structure = xray.structure(
crystal_symmetry=crystal.symmetry(
unit_cell=unit_cell_lengths,
space_group_symbol="P112"))
for serial,site in zip(count(1), sites_cart):
xray_structure.add_scatterer(xray.scatterer(
label="C%02d"%serial,
site=xray_structure.unit_cell().fractionalize(site)))
if (0 or verbose):
xray_structure.show_summary().show_scatterers()
p1_structure = (xray_structure
.apply_shift((-.5,-.5,0))
.expand_to_p1()
.apply_shift((.5,.5,0)))
for shift in [(1,0,0), (0,1,0), (0,0,1)]:
p1_structure.add_scatterers(p1_structure.apply_shift(shift).scatterers())
if (0 or verbose):
open("p1_structure.pdb", "w").write(p1_structure.as_pdb_file())
nonbonded_cutoff = 6.5
asu_mappings = xray_structure.asu_mappings(
buffer_thickness=nonbonded_cutoff)
bond_asu_table = crystal.pair_asu_table(asu_mappings=asu_mappings)
geometry_restraints.add_pairs(bond_asu_table, bond_proxies.simple)
shell_asu_tables = crystal.coordination_sequences.shell_asu_tables(
pair_asu_table=bond_asu_table,
max_shell=3)
shell_sym_tables = [shell_asu_table.extract_pair_sym_table()
for shell_asu_table in shell_asu_tables]
bond_params_table = geometry_restraints.extract_bond_params(
n_seq=sites_cart.size(),
bond_simple_proxies=bond_proxies.simple)
atom_energy_types = flex.std_string(sites_cart.size(), "Default")
nonbonded_params = geometry_restraints.nonbonded_params(
factor_1_4_interactions=vdw_1_4_factor,
const_shrink_1_4_interactions=0,
default_distance=default_vdw_distance)
nonbonded_params.distance_table.setdefault(
"Default")["Default"] = default_vdw_distance
pair_proxies = geometry_restraints.pair_proxies(
bond_params_table=bond_params_table,
shell_asu_tables=shell_asu_tables,
model_indices=None,
conformer_indices=None,
nonbonded_params=nonbonded_params,
nonbonded_types=atom_energy_types,
nonbonded_distance_cutoff_plus_buffer=nonbonded_cutoff)
if (0 or verbose):
print "pair_proxies.bond_proxies.n_total():", \
pair_proxies.bond_proxies.n_total(),
print "simple:", pair_proxies.bond_proxies.simple.size(),
print "sym:", pair_proxies.bond_proxies.asu.size()
print "pair_proxies.nonbonded_proxies.n_total():", \
pair_proxies.nonbonded_proxies.n_total(),
print "simple:", pair_proxies.nonbonded_proxies.simple.size(),
print "sym:", pair_proxies.nonbonded_proxies.asu.size()
print "min_distance_nonbonded: %.2f" % flex.min(
geometry_restraints.nonbonded_deltas(
sites_cart=sites_cart,
sorted_asu_proxies=pair_proxies.nonbonded_proxies))
s = StringIO()
pair_proxies.bond_proxies.show_histogram_of_model_distances(
sites_cart=sites_cart,
f=s,
prefix="[]")
assert s.getvalue().splitlines()[0] == "[]Histogram of bond lengths:"
assert s.getvalue().splitlines()[5].startswith("[] 1.80 - 1.80:")
s = StringIO()
pair_proxies.bond_proxies.show_histogram_of_deltas(
sites_cart=sites_cart,
f=s,
prefix="][")
assert s.getvalue().splitlines()[0] == "][Histogram of bond deltas:"
assert s.getvalue().splitlines()[5].startswith("][ 0.000 - 0.000:")
s = StringIO()
pair_proxies.bond_proxies.show_sorted(
by_value="residual",
sites_cart=sites_cart,
max_items=3,
f=s,
prefix=":;")
l = s.getvalue().splitlines()
assert l[0] == ":;Bond restraints: 18"
assert l[1] == ":;Sorted by residual:"
assert l[2].startswith(":;bond ")
assert l[3].startswith(":; ")
assert l[4] == ":; ideal model delta sigma weight residual"
for i in [5,-2]:
assert l[i].startswith(":; 1.800 1.800 ")
assert l[-1] == ":;... (remaining 15 not shown)"
s = StringIO()
pair_proxies.nonbonded_proxies.show_histogram_of_model_distances(
sites_cart=sites_cart,
f=s,
prefix="]^")
assert not show_diff(s.getvalue(), """\
]^Histogram of nonbonded interaction distances:
]^ 2.16 - 3.03: 3
]^ 3.03 - 3.89: 12
]^ 3.89 - 4.75: 28
]^ 4.75 - 5.61: 44
]^ 5.61 - 6.48: 54
""")
s = StringIO()
pair_proxies.nonbonded_proxies.show_sorted(
by_value="delta",
sites_cart=sites_cart,
max_items=7,
f=s,
prefix=">,")
assert not show_diff(s.getvalue(), """\
>,Nonbonded interactions: 141
>,Sorted by model distance:
>,nonbonded 15
>, 15
>, model vdw sym.op.
>, 2.164 3.600 -x+2,-y+1,z
...
>,nonbonded 4
>, 8
>, model vdw
>, 3.414 3.600
>,... (remaining 134 not shown)
""",
selections=[range(6), range(-5,0)])
vdw_1_sticks = []
vdw_2_sticks = []
for proxy in pair_proxies.nonbonded_proxies.simple:
if (proxy.vdw_distance == default_vdw_distance):
vdw_1_sticks.append(pml_stick(
begin=sites_cart[proxy.i_seqs[0]],
end=sites_cart[proxy.i_seqs[1]]))
else:
vdw_2_sticks.append(pml_stick(
begin=sites_cart[proxy.i_seqs[0]],
end=sites_cart[proxy.i_seqs[1]]))
mps = asu_mappings.mappings()
for proxy in pair_proxies.nonbonded_proxies.asu:
if (proxy.vdw_distance == default_vdw_distance):
vdw_1_sticks.append(pml_stick(
begin=mps[proxy.i_seq][0].mapped_site(),
end=mps[proxy.j_seq][proxy.j_sym].mapped_site()))
else:
vdw_2_sticks.append(pml_stick(
begin=mps[proxy.i_seq][0].mapped_site(),
end=mps[proxy.j_seq][proxy.j_sym].mapped_site()))
if (0 or verbose):
pml_write(f=open("vdw_1.pml", "w"), label="vdw_1", sticks=vdw_1_sticks)
pml_write(f=open("vdw_2.pml", "w"), label="vdw_2", sticks=vdw_2_sticks)
#
i_pdb = count(2)
for use_crystal_symmetry in [False, True]:
if (not use_crystal_symmetry):
crystal_symmetry = None
site_symmetry_table = None
else:
crystal_symmetry = xray_structure
site_symmetry_table = xray_structure.site_symmetry_table()
for sites_cart in [sites_cart_manual.deep_copy(),
sites_cart_minimized_1.deep_copy()]:
manager = geometry_restraints.manager.manager(
crystal_symmetry=crystal_symmetry,
site_symmetry_table=site_symmetry_table,
nonbonded_params=nonbonded_params,
nonbonded_types=atom_energy_types,
nonbonded_function=geometry_restraints.prolsq_repulsion_function(),
bond_params_table=bond_params_table,
shell_sym_tables=shell_sym_tables,
nonbonded_distance_cutoff=nonbonded_cutoff,
nonbonded_buffer=1,
angle_proxies=angle_proxies,
plain_pairs_radius=5)
manager = manager.select(selection=flex.bool(sites_cart.size(), True))
manager = manager.select(
iselection=flex.size_t_range(stop=sites_cart.size()))
pair_proxies = manager.pair_proxies(sites_cart=sites_cart)
minimized = geometry_restraints.lbfgs.lbfgs(
sites_cart=sites_cart,
geometry_restraints_manager=manager,
lbfgs_termination_params=scitbx.lbfgs.termination_parameters(
max_iterations=1000))
if (0 or verbose):
minimized.final_target_result.show()
print "number of function evaluations:", minimized.minimizer.nfun()
print "n_updates_pair_proxies:", manager.n_updates_pair_proxies
if (not use_crystal_symmetry):
assert minimized.final_target_result.bond_residual_sum < 1.e-3
assert minimized.final_target_result.nonbonded_residual_sum < 0.1
else:
assert minimized.final_target_result.bond_residual_sum < 1.e-2
assert minimized.final_target_result.nonbonded_residual_sum < 0.1
assert minimized.final_target_result.angle_residual_sum < 1.e-3
if (0 or verbose):
pdb_file_name = "minimized_%d.pdb" % i_pdb.next()
print "Writing file:", pdb_file_name
dump_pdb(file_name=pdb_file_name, sites_cart=sites_cart)
if (manager.site_symmetry_table is None):
additional_site_symmetry_table = None
else:
additional_site_symmetry_table = sgtbx.site_symmetry_table()
assert manager.new_including_isolated_sites(
n_additional_sites=0,
site_symmetry_table=additional_site_symmetry_table,
nonbonded_types=flex.std_string()).plain_pairs_radius \
== manager.plain_pairs_radius
if (crystal_symmetry is not None):
assert len(manager.plain_pair_sym_table) == 16
if (0 or verbose):
manager.plain_pair_sym_table.show()
#
xray_structure.set_u_iso(values=flex.double([
0.77599982480241358, 0.38745781137212021, 0.20667558236418682,
0.99759840171302094, 0.8917287406687805, 0.64780251325379845,
0.24878590382983534, 0.59480621182194615, 0.58695637792905142,
0.33997130213653637, 0.51258699130743735, 0.79760289141276675,
0.39996577657875021, 0.4329328819341467, 0.70422156561726479,
0.87260110626999332]))
class parameters: pass
parameters.sphere_radius = 5
parameters.distance_power = 0.7
parameters.average_power = 0.9
parameters.wilson_b_weight = 1.3952
parameters.wilson_b_weight_auto = False
adp_energies = adp_restraints.energies_iso(
geometry_restraints_manager=manager,
xray_structure=xray_structure,
parameters=parameters,
wilson_b=None,
use_hd=False,
use_u_local_only = False,
compute_gradients=False,
gradients=None,
normalization=False,
collect=True)
assert adp_energies.number_of_restraints == 69
assert approx_equal(adp_energies.residual_sum, 6.24865382467)
assert adp_energies.gradients is None
assert adp_energies.u_i.size() == adp_energies.number_of_restraints
assert adp_energies.u_j.size() == adp_energies.number_of_restraints
assert adp_energies.r_ij.size() == adp_energies.number_of_restraints
for wilson_b in [None, 10, 100]:
finite_difference_gradients = flex.double()
eps = 1.e-6
for i_scatterer in xrange(xray_structure.scatterers().size()):
rs = []
for signed_eps in [eps, -eps]:
xray_structure_eps = xray_structure.deep_copy_scatterers()
xray_structure_eps.scatterers()[i_scatterer].u_iso += signed_eps
adp_energies = adp_restraints.energies_iso(
geometry_restraints_manager=manager,
xray_structure=xray_structure_eps,
parameters=parameters,
wilson_b=wilson_b,
use_u_local_only = False,
use_hd=False,
compute_gradients=True,
gradients=None,
normalization=False,
collect=False)
rs.append(adp_energies.residual_sum)
assert adp_energies.gradients.size() \
== xray_structure.scatterers().size()
assert adp_energies.u_i == None
assert adp_energies.u_j == None
assert adp_energies.r_ij == None
finite_difference_gradients.append((rs[0]-rs[1])/(2*eps))
sel = flex.bool(xray_structure.scatterers().size(), True)
xray_structure.scatterers().flags_set_grad_u_iso(sel.iselection())
adp_energies = adp_restraints.energies_iso(
geometry_restraints_manager=manager,
xray_structure=xray_structure,
parameters=parameters,
wilson_b=wilson_b,
use_u_local_only = False,
use_hd=False,
compute_gradients=True,
gradients=None,
normalization=False,
collect=False)
assert approx_equal(adp_energies.gradients, finite_difference_gradients)
print "OK"
