def do_exercise(self, verbose=False):
xs = self.xs
indices = self.miller_indices(xs.space_group_info())
f = structure_factors.f_calc_modulus(xs)
f_sq = structure_factors.f_calc_modulus_squared(xs)
for h in indices:
f.linearise(h)
f_sq.linearise(h)
assert approx_equal_relatively(f.observable ** 2, f_sq.observable, relative_error=1e-10)
grad_f_sq = f_sq.grad_observable
two_f_grad_f = 2 * f.observable * f.grad_observable
flex.compare_derivatives(two_f_grad_f, grad_f_sq, eps=1e-12)
def do_exercise(self, verbose=False):
xs = self.xs
indices = self.miller_indices(xs.space_group_info())
f = structure_factors.f_calc_modulus(xs)
f_sq = structure_factors.f_calc_modulus_squared(xs)
for h in indices:
f.linearise(h)
f_sq.linearise(h)
assert approx_equal_relatively(f.observable**2,
f_sq.observable,
relative_error=1e-10)
grad_f_sq = f_sq.grad_observable
two_f_grad_f = (2 * f.observable * f.grad_observable)
flex.compare_derivatives(two_f_grad_f, grad_f_sq, eps=1e-12)
def do_exercise(self, verbose=False):
xs = self.xs
sg = xs.space_group_info().group()
origin_centric_case = sg.is_origin_centric()
indices = self.miller_indices(xs.space_group_info())
f = structure_factors.f_calc_modulus_squared(xs)
f1 = structure_factors.f_calc_modulus_squared(xs)
for h in indices:
f.linearise(h)
fl = f.f_calc
f1.evaluate(h)
fe = f1.f_calc
assert f1.grad_f_calc is None
assert approx_equal_relatively(fe, fl,
relative_error=1e-12), (fe, fl)
if (xs.space_group().is_origin_centric()
and not self.inelastic_scattering):
for h in indices:
f.linearise(h)
assert f.f_calc.imag == 0
assert flex.imag(f.grad_f_calc).all_eq(0)
eta = 1e-8
xs_forward = xs.deep_copy_scatterers()
f_forward = structure_factors.f_calc_modulus_squared(xs_forward)
deltas = flex.double()
for direction in islice(self.structures_forward(xs, xs_forward, eta),
self.n_directions):
for h in indices:
f.linearise(h)
assert approx_equal(abs(f.f_calc)**2, f.observable)
f_forward.linearise(h)
diff_num = (f_forward.observable - f.observable) / eta
diff = f.grad_observable.dot(direction)
delta = abs(1 - diff / diff_num)
deltas.append(delta)
stats = median_statistics(deltas)
tol = 1e-5
assert stats.median < tol, (xs.space_group_info().symbol_and_number(),
stats.median)
assert stats.median_absolute_deviation < tol, (
xs.space_group_info().symbol_and_number(),
stats.median_absolute_deviation)
def exercise_gradient(f_c_in_p1, f_o, flags, n_sampled):
""" where we use finite differences to check our derivatives """
from random import random
from scitbx.math import approx_equal_relatively
f_o_sq = f_o.as_intensity_array()
for i in xrange(n_sampled):
x = mat.col((random(), random(), random()))
gos = symmetrised_shifted_structure_factors(
f_o_sq, f_c_in_p1, x, compute_gradient=True).misfit(f_o_sq)
for j,e in enumerate([ (1,0,0), (0,1,0), (0,0,1) ]):
h = mat.col(e)*1e-3
fm3, fm2, fm1, f1, f2, f3 = [
symmetrised_shifted_structure_factors(
f_o_sq, f_c_in_p1, y).misfit(f_o_sq).value
for y in (x-3*h, x-2*h, x-h, x+h, x+2*h, x+3*h) ]
finite_diff = ((f3 - fm3)/60 - 3/20*(f2 - fm2) + 3/4*(f1 - fm1))/abs(h)
assert approx_equal_relatively(gos.gradient[j], finite_diff,
relative_error=1e-2,
near_zero_threshold=1e-6), \
(j, i, tuple(x), gos.gradient[j], finite_diff)
def do_exercise(self, verbose=False):
xs = self.xs
sg = xs.space_group_info().group()
origin_centric_case = sg.is_origin_centric()
indices = self.miller_indices(xs.space_group_info())
f = structure_factors.f_calc_modulus_squared(xs)
f1 = structure_factors.f_calc_modulus_squared(xs)
for h in indices:
f.linearise(h)
fl = f.f_calc
f1.evaluate(h)
fe = f1.f_calc
assert f1.grad_f_calc is None
assert approx_equal_relatively(fe, fl, relative_error=1e-12), (fe, fl)
if xs.space_group().is_origin_centric() and not self.inelastic_scattering:
for h in indices:
f.linearise(h)
assert f.f_calc.imag == 0
assert flex.imag(f.grad_f_calc).all_eq(0)
eta = 1e-8
xs_forward = xs.deep_copy_scatterers()
f_forward = structure_factors.f_calc_modulus_squared(xs_forward)
deltas = flex.double()
for direction in islice(self.structures_forward(xs, xs_forward, eta), self.n_directions):
for h in indices:
f.linearise(h)
assert approx_equal(abs(f.f_calc) ** 2, f.observable)
f_forward.linearise(h)
diff_num = (f_forward.observable - f.observable) / eta
diff = f.grad_observable.dot(direction)
delta = abs(1 - diff / diff_num)
deltas.append(delta)
stats = median_statistics(deltas)
tol = 1e-3
assert stats.median < tol, (str(space_group_info), stats.median)
assert stats.median_absolute_deviation < tol, (str(space_group_info), stats.median_absolute_deviation)
def exercise_gradient(f_c_in_p1, f_o, flags, n_sampled):
""" where we use finite differences to check our derivatives """
from random import random
from scitbx.math import approx_equal_relatively
f_o_sq = f_o.as_intensity_array()
for i in xrange(n_sampled):
x = mat.col((random(), random(), random()))
gos = symmetrised_shifted_structure_factors(
f_o_sq, f_c_in_p1, x, compute_gradient=True).misfit(f_o_sq)
for j, e in enumerate([(1, 0, 0), (0, 1, 0), (0, 0, 1)]):
h = mat.col(e) * 1e-3
fm3, fm2, fm1, f1, f2, f3 = [
symmetrised_shifted_structure_factors(f_o_sq, f_c_in_p1,
y).misfit(f_o_sq).value
for y in (x - 3 * h, x - 2 * h, x - h, x + h, x + 2 * h,
x + 3 * h)
]
finite_diff = ((f3 - fm3) / 60 - 3 / 20 * (f2 - fm2) + 3 / 4 *
(f1 - fm1)) / abs(h)
assert approx_equal_relatively(gos.gradient[j], finite_diff,
relative_error=1e-2,
near_zero_threshold=1e-6), \
(j, i, tuple(x), gos.gradient[j], finite_diff)
def calculate_scaling(self,
miller_array,
convergence_crit_perc=0.01,
convergence_reject_perc=97.5,
max_iter=20):
"""Calculate the scaling between two arrays"""
assert convergence_reject_perc > 90.0
# Convert to intensities and extract d_star_sq
new_miller = miller_array.as_intensity_array()
new_kernel = self._kernel_normalisation(miller_array=new_miller)
# Calculate new range of d_star_sq
d_star_sq_min, d_star_sq_max = self._common_d_star_sq_range(
d_star_sq=new_kernel.d_star_sq_array)
# Create interpolator for the two arrays (new and reference)
interpolator = scale_curves.curve_interpolator(d_star_sq_min,
d_star_sq_max,
self._npoints)
# Interpolate the two curves (use full range of the two array)
new_itpl_d_star_sq, new_itpl_mean_I, dummy, dummy = interpolator.interpolate(
x_array=new_kernel.d_star_sq_array,
y_array=new_kernel.mean_I_array)
ref_itpl_d_star_sq, ref_itpl_mean_I, dummy, dummy = interpolator.interpolate(
x_array=self.ref_kernel.d_star_sq_array,
y_array=self.ref_kernel.mean_I_array)
# Initalise convergence loop - begin by scaling over all points
selection = flex.bool(self._npoints, True)
# Set initial scale factor to small value
curr_b = 1e-6
# Percent change between iterations - convergence when delta <convergence_criterion
n_iter = 0
# Report in case of error
report = Report('Scaling log:', verbose=False)
while n_iter < max_iter:
report('---')
report('ITER: ' + str(n_iter))
if selection.all_eq(False):
print("Selection now empty, breaking")
break
# Run optimisation on the linear scaling
lsc = ExponentialScaling(x_values=interpolator.target_x,
ref_values=ref_itpl_mean_I,
scl_values=new_itpl_mean_I,
weights=selection.as_double())
# Calculate scaling B-factor
lsc.scaling_b_factor = -0.5 * list(lsc.optimised_values)[0]
# Break if fitted to 0
if approx_equal_relatively(0.0, lsc.scaling_b_factor, 1e-6):
report('Scaling is approximately 0.0 - stopping')
break
# Calculate percentage change
report('Curr/New: ' + str(curr_b) + '\t' +
str(lsc.scaling_b_factor))
delta = abs((curr_b - lsc.scaling_b_factor) / curr_b)
report('Delta: ' + str(delta))
if delta < convergence_crit_perc:
report('Scaling has converged to within tolerance - stopping')
break
# Update selection
report('Curr Selection Size: ' + str(sum(selection)))
ref_diffs = flex.log(lsc.ref_values) - flex.log(lsc.out_values)
#abs_diffs = flex.abs(ref_diffs)
sel_diffs = ref_diffs.select(selection)
rej_val_high = numpy.percentile(sel_diffs, convergence_reject_perc)
rej_val_low = numpy.percentile(sel_diffs,
100.0 - convergence_reject_perc)
report('Percentile: ' + str(convergence_reject_perc) + '\t<' +
str(rej_val_low) + '\t>' + str(rej_val_high))
selection.set_selected(ref_diffs > rej_val_high, False)
selection.set_selected(ref_diffs < rej_val_low, False)
report('New Selection Size: ' + str(sum(selection)))
# Update loop params
curr_b = lsc.scaling_b_factor
n_iter += 1
lsc.unscaled_ln_rmsd = (flex.log(lsc.ref_values) - flex.log(
lsc.scl_values)).norm() / (lsc.ref_values.size()**0.5)
lsc.scaled_ln_rmsd = (flex.log(lsc.ref_values) - flex.log(
lsc.out_values)).norm() / (lsc.ref_values.size()**0.5)
lsc.unscaled_ln_dev = flex.sum(
flex.abs(flex.log(lsc.ref_values) - flex.log(lsc.scl_values)))
lsc.scaled_ln_dev = flex.sum(
flex.abs(flex.log(lsc.ref_values) - flex.log(lsc.out_values)))
return lsc
def structure_factors(self, max_cycles=10):
"""P. van der Sluis and A. L. Spek, Acta Cryst. (1990). A46, 194-201."""
assert self.mask is not None
if self.n_voids() == 0: return
if self.use_set_completion:
f_calc_set = self.complete_set
else:
f_calc_set = self.fo2.set()
self.f_calc = f_calc_set.structure_factors_from_scatterers(
self.xray_structure, algorithm="direct").f_calc()
f_obs = self.f_obs()
self.scale_factor = flex.sum(f_obs.data())/flex.sum(
flex.abs(self.f_calc.data()))
f_obs_minus_f_calc = f_obs.f_obs_minus_f_calc(
1/self.scale_factor, self.f_calc)
self.fft_scale = self.xray_structure.unit_cell().volume()\
/ self.crystal_gridding.n_grid_points()
epsilon_for_min_residual = 2
for i in range(max_cycles):
self.diff_map = miller.fft_map(self.crystal_gridding, f_obs_minus_f_calc)
self.diff_map.apply_volume_scaling()
stats = self.diff_map.statistics()
masked_diff_map = self.diff_map.real_map_unpadded().set_selected(
self.mask.data.as_double() == 0, 0)
n_solvent_grid_points = self.n_solvent_grid_points()
for j in range(self.n_voids()):
# exclude voids with negative electron counts from the masked map
# set the electron density in those areas to be zero
selection = self.mask.data == j+2
if self.exclude_void_flags[j]:
masked_diff_map.set_selected(selection, 0)
continue
diff_map_ = masked_diff_map.deep_copy().set_selected(~selection, 0)
f_000 = flex.sum(diff_map_) * self.fft_scale
f_000_s = f_000 * (
self.crystal_gridding.n_grid_points() /
(self.crystal_gridding.n_grid_points() - n_solvent_grid_points))
if f_000_s < 0:
masked_diff_map.set_selected(selection, 0)
f_000_s = 0
self.exclude_void_flags[j] = True
self.f_000 = flex.sum(masked_diff_map) * self.fft_scale
f_000_s = self.f_000 * (masked_diff_map.size() /
(masked_diff_map.size() - self.n_solvent_grid_points()))
if (self.f_000_s is not None and
approx_equal_relatively(self.f_000_s, f_000_s, 0.0001)):
break # we have reached convergence
else:
self.f_000_s = f_000_s
masked_diff_map.add_selected(
self.mask.data.as_double() > 0,
self.f_000_s/self.xray_structure.unit_cell().volume())
if 0:
from crys3d import wx_map_viewer
wx_map_viewer.display(
title="masked diff_map",
raw_map=masked_diff_map.as_double(),
unit_cell=f_obs.unit_cell())
self._f_mask = f_obs.structure_factors_from_map(map=masked_diff_map)
self._f_mask *= self.fft_scale
scales = []
residuals = []
min_residual = 1000
for epsilon in xfrange(epsilon_for_min_residual, 0.9, -0.2):
f_model_ = self.f_model(epsilon=epsilon)
scale = flex.sum(f_obs.data())/flex.sum(flex.abs(f_model_.data()))
residual = flex.sum(flex.abs(
1/scale * flex.abs(f_obs.data())- flex.abs(f_model_.data()))) \
/ flex.sum(1/scale * flex.abs(f_obs.data()))
scales.append(scale)
residuals.append(residual)
min_residual = min(min_residual, residual)
if min_residual == residual:
scale_for_min_residual = scale
epsilon_for_min_residual = epsilon
self.scale_factor = scale_for_min_residual
f_model = self.f_model(epsilon=epsilon_for_min_residual)
f_obs = self.f_obs()
f_obs_minus_f_calc = f_obs.phase_transfer(f_model).f_obs_minus_f_calc(
1/self.scale_factor, self.f_calc)
return self._f_mask
def structure_factors(self, max_cycles=10):
"""P. van der Sluis and A. L. Spek, Acta Cryst. (1990). A46, 194-201."""
assert self.mask is not None
if self.n_voids() == 0:
return
if self.use_set_completion:
f_calc_set = self.complete_set
else:
f_calc_set = self.fo2.set()
self.f_calc = f_calc_set.structure_factors_from_scatterers(self.xray_structure, algorithm="direct").f_calc()
f_obs = self.f_obs()
self.scale_factor = flex.sum(f_obs.data()) / flex.sum(flex.abs(self.f_calc.data()))
f_obs_minus_f_calc = f_obs.f_obs_minus_f_calc(1 / self.scale_factor, self.f_calc)
self.fft_scale = self.xray_structure.unit_cell().volume() / self.crystal_gridding.n_grid_points()
epsilon_for_min_residual = 2
for i in range(max_cycles):
self.diff_map = miller.fft_map(self.crystal_gridding, f_obs_minus_f_calc)
self.diff_map.apply_volume_scaling()
stats = self.diff_map.statistics()
masked_diff_map = self.diff_map.real_map_unpadded().set_selected(self.mask.data.as_double() == 0, 0)
n_solvent_grid_points = self.n_solvent_grid_points()
for j in range(self.n_voids()):
# exclude voids with negative electron counts from the masked map
# set the electron density in those areas to be zero
selection = self.mask.data == j + 2
if self.exclude_void_flags[j]:
masked_diff_map.set_selected(selection, 0)
continue
diff_map_ = masked_diff_map.deep_copy().set_selected(~selection, 0)
f_000 = flex.sum(diff_map_) * self.fft_scale
f_000_s = f_000 * (
self.crystal_gridding.n_grid_points()
/ (self.crystal_gridding.n_grid_points() - n_solvent_grid_points)
)
if f_000_s < 0:
masked_diff_map.set_selected(selection, 0)
f_000_s = 0
self.exclude_void_flags[j] = True
self.f_000 = flex.sum(masked_diff_map) * self.fft_scale
f_000_s = self.f_000 * (masked_diff_map.size() / (masked_diff_map.size() - self.n_solvent_grid_points()))
if self.f_000_s is not None and approx_equal_relatively(self.f_000_s, f_000_s, 0.0001):
break # we have reached convergence
else:
self.f_000_s = f_000_s
masked_diff_map.add_selected(
self.mask.data.as_double() > 0, self.f_000_s / self.xray_structure.unit_cell().volume()
)
if 0:
from crys3d import wx_map_viewer
wx_map_viewer.display(
title="masked diff_map", raw_map=masked_diff_map.as_double(), unit_cell=f_obs.unit_cell()
)
self._f_mask = f_obs.structure_factors_from_map(map=masked_diff_map)
self._f_mask *= self.fft_scale
scales = []
residuals = []
min_residual = 1000
for epsilon in xfrange(epsilon_for_min_residual, 0.9, -0.2):
f_model_ = self.f_model(epsilon=epsilon)
scale = flex.sum(f_obs.data()) / flex.sum(flex.abs(f_model_.data()))
residual = flex.sum(
flex.abs(1 / scale * flex.abs(f_obs.data()) - flex.abs(f_model_.data()))
) / flex.sum(1 / scale * flex.abs(f_obs.data()))
scales.append(scale)
residuals.append(residual)
min_residual = min(min_residual, residual)
if min_residual == residual:
scale_for_min_residual = scale
epsilon_for_min_residual = epsilon
self.scale_factor = scale_for_min_residual
f_model = self.f_model(epsilon=epsilon_for_min_residual)
f_obs = self.f_obs()
f_obs_minus_f_calc = f_obs.phase_transfer(f_model).f_obs_minus_f_calc(1 / self.scale_factor, self.f_calc)
return self._f_mask
