def do_exercise(self, verbose=False): xs = self.xs indices = self.miller_indices(xs.space_group_info()) f = structure_factors.f_calc_modulus(xs) f_sq = structure_factors.f_calc_modulus_squared(xs) for h in indices: f.linearise(h) f_sq.linearise(h) assert approx_equal_relatively(f.observable ** 2, f_sq.observable, relative_error=1e-10) grad_f_sq = f_sq.grad_observable two_f_grad_f = 2 * f.observable * f.grad_observable flex.compare_derivatives(two_f_grad_f, grad_f_sq, eps=1e-12)
def do_exercise(self, verbose=False): xs = self.xs indices = self.miller_indices(xs.space_group_info()) f = structure_factors.f_calc_modulus(xs) f_sq = structure_factors.f_calc_modulus_squared(xs) for h in indices: f.linearise(h) f_sq.linearise(h) assert approx_equal_relatively(f.observable**2, f_sq.observable, relative_error=1e-10) grad_f_sq = f_sq.grad_observable two_f_grad_f = (2 * f.observable * f.grad_observable) flex.compare_derivatives(two_f_grad_f, grad_f_sq, eps=1e-12)
def do_exercise(self, verbose=False): xs = self.xs sg = xs.space_group_info().group() origin_centric_case = sg.is_origin_centric() indices = self.miller_indices(xs.space_group_info()) f = structure_factors.f_calc_modulus_squared(xs) f1 = structure_factors.f_calc_modulus_squared(xs) for h in indices: f.linearise(h) fl = f.f_calc f1.evaluate(h) fe = f1.f_calc assert f1.grad_f_calc is None assert approx_equal_relatively(fe, fl, relative_error=1e-12), (fe, fl) if (xs.space_group().is_origin_centric() and not self.inelastic_scattering): for h in indices: f.linearise(h) assert f.f_calc.imag == 0 assert flex.imag(f.grad_f_calc).all_eq(0) eta = 1e-8 xs_forward = xs.deep_copy_scatterers() f_forward = structure_factors.f_calc_modulus_squared(xs_forward) deltas = flex.double() for direction in islice(self.structures_forward(xs, xs_forward, eta), self.n_directions): for h in indices: f.linearise(h) assert approx_equal(abs(f.f_calc)**2, f.observable) f_forward.linearise(h) diff_num = (f_forward.observable - f.observable) / eta diff = f.grad_observable.dot(direction) delta = abs(1 - diff / diff_num) deltas.append(delta) stats = median_statistics(deltas) tol = 1e-5 assert stats.median < tol, (xs.space_group_info().symbol_and_number(), stats.median) assert stats.median_absolute_deviation < tol, ( xs.space_group_info().symbol_and_number(), stats.median_absolute_deviation)
def exercise_gradient(f_c_in_p1, f_o, flags, n_sampled): """ where we use finite differences to check our derivatives """ from random import random from scitbx.math import approx_equal_relatively f_o_sq = f_o.as_intensity_array() for i in xrange(n_sampled): x = mat.col((random(), random(), random())) gos = symmetrised_shifted_structure_factors( f_o_sq, f_c_in_p1, x, compute_gradient=True).misfit(f_o_sq) for j,e in enumerate([ (1,0,0), (0,1,0), (0,0,1) ]): h = mat.col(e)*1e-3 fm3, fm2, fm1, f1, f2, f3 = [ symmetrised_shifted_structure_factors( f_o_sq, f_c_in_p1, y).misfit(f_o_sq).value for y in (x-3*h, x-2*h, x-h, x+h, x+2*h, x+3*h) ] finite_diff = ((f3 - fm3)/60 - 3/20*(f2 - fm2) + 3/4*(f1 - fm1))/abs(h) assert approx_equal_relatively(gos.gradient[j], finite_diff, relative_error=1e-2, near_zero_threshold=1e-6), \ (j, i, tuple(x), gos.gradient[j], finite_diff)
def do_exercise(self, verbose=False): xs = self.xs sg = xs.space_group_info().group() origin_centric_case = sg.is_origin_centric() indices = self.miller_indices(xs.space_group_info()) f = structure_factors.f_calc_modulus_squared(xs) f1 = structure_factors.f_calc_modulus_squared(xs) for h in indices: f.linearise(h) fl = f.f_calc f1.evaluate(h) fe = f1.f_calc assert f1.grad_f_calc is None assert approx_equal_relatively(fe, fl, relative_error=1e-12), (fe, fl) if xs.space_group().is_origin_centric() and not self.inelastic_scattering: for h in indices: f.linearise(h) assert f.f_calc.imag == 0 assert flex.imag(f.grad_f_calc).all_eq(0) eta = 1e-8 xs_forward = xs.deep_copy_scatterers() f_forward = structure_factors.f_calc_modulus_squared(xs_forward) deltas = flex.double() for direction in islice(self.structures_forward(xs, xs_forward, eta), self.n_directions): for h in indices: f.linearise(h) assert approx_equal(abs(f.f_calc) ** 2, f.observable) f_forward.linearise(h) diff_num = (f_forward.observable - f.observable) / eta diff = f.grad_observable.dot(direction) delta = abs(1 - diff / diff_num) deltas.append(delta) stats = median_statistics(deltas) tol = 1e-3 assert stats.median < tol, (str(space_group_info), stats.median) assert stats.median_absolute_deviation < tol, (str(space_group_info), stats.median_absolute_deviation)
def exercise_gradient(f_c_in_p1, f_o, flags, n_sampled): """ where we use finite differences to check our derivatives """ from random import random from scitbx.math import approx_equal_relatively f_o_sq = f_o.as_intensity_array() for i in xrange(n_sampled): x = mat.col((random(), random(), random())) gos = symmetrised_shifted_structure_factors( f_o_sq, f_c_in_p1, x, compute_gradient=True).misfit(f_o_sq) for j, e in enumerate([(1, 0, 0), (0, 1, 0), (0, 0, 1)]): h = mat.col(e) * 1e-3 fm3, fm2, fm1, f1, f2, f3 = [ symmetrised_shifted_structure_factors(f_o_sq, f_c_in_p1, y).misfit(f_o_sq).value for y in (x - 3 * h, x - 2 * h, x - h, x + h, x + 2 * h, x + 3 * h) ] finite_diff = ((f3 - fm3) / 60 - 3 / 20 * (f2 - fm2) + 3 / 4 * (f1 - fm1)) / abs(h) assert approx_equal_relatively(gos.gradient[j], finite_diff, relative_error=1e-2, near_zero_threshold=1e-6), \ (j, i, tuple(x), gos.gradient[j], finite_diff)
def calculate_scaling(self, miller_array, convergence_crit_perc=0.01, convergence_reject_perc=97.5, max_iter=20): """Calculate the scaling between two arrays""" assert convergence_reject_perc > 90.0 # Convert to intensities and extract d_star_sq new_miller = miller_array.as_intensity_array() new_kernel = self._kernel_normalisation(miller_array=new_miller) # Calculate new range of d_star_sq d_star_sq_min, d_star_sq_max = self._common_d_star_sq_range( d_star_sq=new_kernel.d_star_sq_array) # Create interpolator for the two arrays (new and reference) interpolator = scale_curves.curve_interpolator(d_star_sq_min, d_star_sq_max, self._npoints) # Interpolate the two curves (use full range of the two array) new_itpl_d_star_sq, new_itpl_mean_I, dummy, dummy = interpolator.interpolate( x_array=new_kernel.d_star_sq_array, y_array=new_kernel.mean_I_array) ref_itpl_d_star_sq, ref_itpl_mean_I, dummy, dummy = interpolator.interpolate( x_array=self.ref_kernel.d_star_sq_array, y_array=self.ref_kernel.mean_I_array) # Initalise convergence loop - begin by scaling over all points selection = flex.bool(self._npoints, True) # Set initial scale factor to small value curr_b = 1e-6 # Percent change between iterations - convergence when delta <convergence_criterion n_iter = 0 # Report in case of error report = Report('Scaling log:', verbose=False) while n_iter < max_iter: report('---') report('ITER: ' + str(n_iter)) if selection.all_eq(False): print("Selection now empty, breaking") break # Run optimisation on the linear scaling lsc = ExponentialScaling(x_values=interpolator.target_x, ref_values=ref_itpl_mean_I, scl_values=new_itpl_mean_I, weights=selection.as_double()) # Calculate scaling B-factor lsc.scaling_b_factor = -0.5 * list(lsc.optimised_values)[0] # Break if fitted to 0 if approx_equal_relatively(0.0, lsc.scaling_b_factor, 1e-6): report('Scaling is approximately 0.0 - stopping') break # Calculate percentage change report('Curr/New: ' + str(curr_b) + '\t' + str(lsc.scaling_b_factor)) delta = abs((curr_b - lsc.scaling_b_factor) / curr_b) report('Delta: ' + str(delta)) if delta < convergence_crit_perc: report('Scaling has converged to within tolerance - stopping') break # Update selection report('Curr Selection Size: ' + str(sum(selection))) ref_diffs = flex.log(lsc.ref_values) - flex.log(lsc.out_values) #abs_diffs = flex.abs(ref_diffs) sel_diffs = ref_diffs.select(selection) rej_val_high = numpy.percentile(sel_diffs, convergence_reject_perc) rej_val_low = numpy.percentile(sel_diffs, 100.0 - convergence_reject_perc) report('Percentile: ' + str(convergence_reject_perc) + '\t<' + str(rej_val_low) + '\t>' + str(rej_val_high)) selection.set_selected(ref_diffs > rej_val_high, False) selection.set_selected(ref_diffs < rej_val_low, False) report('New Selection Size: ' + str(sum(selection))) # Update loop params curr_b = lsc.scaling_b_factor n_iter += 1 lsc.unscaled_ln_rmsd = (flex.log(lsc.ref_values) - flex.log( lsc.scl_values)).norm() / (lsc.ref_values.size()**0.5) lsc.scaled_ln_rmsd = (flex.log(lsc.ref_values) - flex.log( lsc.out_values)).norm() / (lsc.ref_values.size()**0.5) lsc.unscaled_ln_dev = flex.sum( flex.abs(flex.log(lsc.ref_values) - flex.log(lsc.scl_values))) lsc.scaled_ln_dev = flex.sum( flex.abs(flex.log(lsc.ref_values) - flex.log(lsc.out_values))) return lsc
def structure_factors(self, max_cycles=10): """P. van der Sluis and A. L. Spek, Acta Cryst. (1990). A46, 194-201.""" assert self.mask is not None if self.n_voids() == 0: return if self.use_set_completion: f_calc_set = self.complete_set else: f_calc_set = self.fo2.set() self.f_calc = f_calc_set.structure_factors_from_scatterers( self.xray_structure, algorithm="direct").f_calc() f_obs = self.f_obs() self.scale_factor = flex.sum(f_obs.data())/flex.sum( flex.abs(self.f_calc.data())) f_obs_minus_f_calc = f_obs.f_obs_minus_f_calc( 1/self.scale_factor, self.f_calc) self.fft_scale = self.xray_structure.unit_cell().volume()\ / self.crystal_gridding.n_grid_points() epsilon_for_min_residual = 2 for i in range(max_cycles): self.diff_map = miller.fft_map(self.crystal_gridding, f_obs_minus_f_calc) self.diff_map.apply_volume_scaling() stats = self.diff_map.statistics() masked_diff_map = self.diff_map.real_map_unpadded().set_selected( self.mask.data.as_double() == 0, 0) n_solvent_grid_points = self.n_solvent_grid_points() for j in range(self.n_voids()): # exclude voids with negative electron counts from the masked map # set the electron density in those areas to be zero selection = self.mask.data == j+2 if self.exclude_void_flags[j]: masked_diff_map.set_selected(selection, 0) continue diff_map_ = masked_diff_map.deep_copy().set_selected(~selection, 0) f_000 = flex.sum(diff_map_) * self.fft_scale f_000_s = f_000 * ( self.crystal_gridding.n_grid_points() / (self.crystal_gridding.n_grid_points() - n_solvent_grid_points)) if f_000_s < 0: masked_diff_map.set_selected(selection, 0) f_000_s = 0 self.exclude_void_flags[j] = True self.f_000 = flex.sum(masked_diff_map) * self.fft_scale f_000_s = self.f_000 * (masked_diff_map.size() / (masked_diff_map.size() - self.n_solvent_grid_points())) if (self.f_000_s is not None and approx_equal_relatively(self.f_000_s, f_000_s, 0.0001)): break # we have reached convergence else: self.f_000_s = f_000_s masked_diff_map.add_selected( self.mask.data.as_double() > 0, self.f_000_s/self.xray_structure.unit_cell().volume()) if 0: from crys3d import wx_map_viewer wx_map_viewer.display( title="masked diff_map", raw_map=masked_diff_map.as_double(), unit_cell=f_obs.unit_cell()) self._f_mask = f_obs.structure_factors_from_map(map=masked_diff_map) self._f_mask *= self.fft_scale scales = [] residuals = [] min_residual = 1000 for epsilon in xfrange(epsilon_for_min_residual, 0.9, -0.2): f_model_ = self.f_model(epsilon=epsilon) scale = flex.sum(f_obs.data())/flex.sum(flex.abs(f_model_.data())) residual = flex.sum(flex.abs( 1/scale * flex.abs(f_obs.data())- flex.abs(f_model_.data()))) \ / flex.sum(1/scale * flex.abs(f_obs.data())) scales.append(scale) residuals.append(residual) min_residual = min(min_residual, residual) if min_residual == residual: scale_for_min_residual = scale epsilon_for_min_residual = epsilon self.scale_factor = scale_for_min_residual f_model = self.f_model(epsilon=epsilon_for_min_residual) f_obs = self.f_obs() f_obs_minus_f_calc = f_obs.phase_transfer(f_model).f_obs_minus_f_calc( 1/self.scale_factor, self.f_calc) return self._f_mask
def structure_factors(self, max_cycles=10): """P. van der Sluis and A. L. Spek, Acta Cryst. (1990). A46, 194-201.""" assert self.mask is not None if self.n_voids() == 0: return if self.use_set_completion: f_calc_set = self.complete_set else: f_calc_set = self.fo2.set() self.f_calc = f_calc_set.structure_factors_from_scatterers(self.xray_structure, algorithm="direct").f_calc() f_obs = self.f_obs() self.scale_factor = flex.sum(f_obs.data()) / flex.sum(flex.abs(self.f_calc.data())) f_obs_minus_f_calc = f_obs.f_obs_minus_f_calc(1 / self.scale_factor, self.f_calc) self.fft_scale = self.xray_structure.unit_cell().volume() / self.crystal_gridding.n_grid_points() epsilon_for_min_residual = 2 for i in range(max_cycles): self.diff_map = miller.fft_map(self.crystal_gridding, f_obs_minus_f_calc) self.diff_map.apply_volume_scaling() stats = self.diff_map.statistics() masked_diff_map = self.diff_map.real_map_unpadded().set_selected(self.mask.data.as_double() == 0, 0) n_solvent_grid_points = self.n_solvent_grid_points() for j in range(self.n_voids()): # exclude voids with negative electron counts from the masked map # set the electron density in those areas to be zero selection = self.mask.data == j + 2 if self.exclude_void_flags[j]: masked_diff_map.set_selected(selection, 0) continue diff_map_ = masked_diff_map.deep_copy().set_selected(~selection, 0) f_000 = flex.sum(diff_map_) * self.fft_scale f_000_s = f_000 * ( self.crystal_gridding.n_grid_points() / (self.crystal_gridding.n_grid_points() - n_solvent_grid_points) ) if f_000_s < 0: masked_diff_map.set_selected(selection, 0) f_000_s = 0 self.exclude_void_flags[j] = True self.f_000 = flex.sum(masked_diff_map) * self.fft_scale f_000_s = self.f_000 * (masked_diff_map.size() / (masked_diff_map.size() - self.n_solvent_grid_points())) if self.f_000_s is not None and approx_equal_relatively(self.f_000_s, f_000_s, 0.0001): break # we have reached convergence else: self.f_000_s = f_000_s masked_diff_map.add_selected( self.mask.data.as_double() > 0, self.f_000_s / self.xray_structure.unit_cell().volume() ) if 0: from crys3d import wx_map_viewer wx_map_viewer.display( title="masked diff_map", raw_map=masked_diff_map.as_double(), unit_cell=f_obs.unit_cell() ) self._f_mask = f_obs.structure_factors_from_map(map=masked_diff_map) self._f_mask *= self.fft_scale scales = [] residuals = [] min_residual = 1000 for epsilon in xfrange(epsilon_for_min_residual, 0.9, -0.2): f_model_ = self.f_model(epsilon=epsilon) scale = flex.sum(f_obs.data()) / flex.sum(flex.abs(f_model_.data())) residual = flex.sum( flex.abs(1 / scale * flex.abs(f_obs.data()) - flex.abs(f_model_.data())) ) / flex.sum(1 / scale * flex.abs(f_obs.data())) scales.append(scale) residuals.append(residual) min_residual = min(min_residual, residual) if min_residual == residual: scale_for_min_residual = scale epsilon_for_min_residual = epsilon self.scale_factor = scale_for_min_residual f_model = self.f_model(epsilon=epsilon_for_min_residual) f_obs = self.f_obs() f_obs_minus_f_calc = f_obs.phase_transfer(f_model).f_obs_minus_f_calc(1 / self.scale_factor, self.f_calc) return self._f_mask