def jacobian_callable(self, values):
# Restore original sigmas
refls = self.scaler.ISIGI
refls['isigi'] = refls['scaled_intensity'] / refls['original_sigmas']
# Propagate errors from postrefinement
self.propagator.error_terms = error_terms.from_x(
values.propagate_terms)
self.propagator.adjust_errors(dI_derrorterms=self.dI_derrorterms,
compute_sums=False)
all_sigmas_normalized, sigma_prime = self.get_normalized_sigmas(values)
# et: error term
df_derrorterms = []
for et, dI_det in zip(
values.propagate_terms,
self.dI_derrorterms[1:]): # don't need dI wrt iobs
dsigmasq_det = 2 * et
dsigmasq_detsq = dI_det**2 * dsigmasq_det
dsigprimesq_detsq = values.SDFACSQ * dsigmasq_detsq
df_detsq = self.df_dpsq(all_sigmas_normalized, sigma_prime,
dsigprimesq_detsq)
df_derrorterms.append(df_detsq)
sdfac_derivatives = super(sdfac_propagate_refinery,
self).jacobian_callable(values)
return sdfac_derivatives + df_derrorterms
def fvec_callable(self, values):
""" Compute the functional by first propagating errors from postrefinement and then
applying the current values for the sd parameters to the input data, then computing
the complete set of normalized deviations and finally using those normalized
deviations to compute the functional."""
# Restore original sigmas
refls = self.ISIGI
refls['isigi'] = refls['scaled_intensity'] / refls['original_sigmas']
# Propagate errors from postrefinement
self.propagator.error_terms = error_terms.from_x(
values.propagate_terms)
self.propagator.adjust_errors(dI_derrorterms=self.dI_derrorterms,
compute_sums=False)
# Apply SD terms
all_sigmas_normalized, _ = self.get_normalized_sigmas(values)
# Compute functional
f = flex.double()
for i, bin in enumerate(self.bins):
binned_normalized_sigmas = all_sigmas_normalized.select(bin)
n = len(binned_normalized_sigmas)
if n == 0:
f.append(0)
continue
# functional is weight * (1-rms(normalized_sigmas))^s summed over all intensitiy bins
f.append(1 - math.sqrt(
flex.mean(binned_normalized_sigmas *
binned_normalized_sigmas)))
return f
def fvec_callable(self, values):
""" Compute the functional by first propagating errors from postrefinement and then
applying the current values for the sd parameters to the input data, then computing
the complete set of normalized deviations and finally using those normalized
deviations to compute the functional."""
# Restore original sigmas
refls = self.ISIGI
refls['isigi'] = refls['scaled_intensity']/refls['original_sigmas']
# Propagate errors from postrefinement
self.propagator.error_terms = error_terms.from_x(values.propagate_terms)
self.propagator.adjust_errors(dI_derrorterms=self.dI_derrorterms, compute_sums=False)
return super(sdfac_propagate_refinery, self).fvec_callable(values)
def adjust_errors(self):
print >> self.log, "Starting adjust_errors"
# Save original sigmas
refls = self.scaler.ISIGI
refls['original_sigmas'] = refls['scaled_intensity'] / refls['isigi']
print >> self.log, "Computing initial estimates of parameters"
propagator = sdfac_propagate(self.scaler, verbose=False)
propagator.initial_estimates()
propagator.adjust_errors(compute_sums=False)
init_params = flex.double(self.get_initial_sdparams_estimates())
init_params.extend(propagator.error_terms.to_x())
values = self.parameterization(init_params)
print >> self.log, "Initial estimates:",
values.show(self.log)
print >> self.log, "Refining error correction parameters"
sels, binned_intensities = self.get_binned_intensities()
minimizer = self.run_minimzer(values, sels)
values = minimizer.get_refined_params()
print >> self.log, "Final",
values.show(self.log)
print >> self.log, "Applying sdfac/sdb/sdadd 1"
# Restore original sigmas
refls['isigi'] = refls['scaled_intensity'] / refls['original_sigmas']
# Propagate refined errors from postrefinement
propagator.error_terms = error_terms.from_x(values.propagate_terms)
propagator.adjust_errors()
minimizer.apply_sd_error_params(self.scaler.ISIGI, values)
self.scaler.summed_weight = flex.double(self.scaler.n_refl, 0.)
self.scaler.summed_wt_I = flex.double(self.scaler.n_refl, 0.)
print >> self.log, "Applying sdfac/sdb/sdadd 2"
for i in xrange(len(self.scaler.ISIGI)):
hkl_id = self.scaler.ISIGI['miller_id'][i]
Intensity = self.scaler.ISIGI['scaled_intensity'][
i] # scaled intensity
sigma = Intensity / self.scaler.ISIGI['isigi'][
i] # corrected sigma
variance = sigma * sigma
self.scaler.summed_wt_I[hkl_id] += Intensity / variance
self.scaler.summed_weight[hkl_id] += 1 / variance
if False:
# validate using http://ccp4wiki.org/~ccp4wiki/wiki/index.php?title=Symmetry%2C_Scale%2C_Merge#Analysis_of_Standard_Deviations
print >> self.log, "Validating"
from matplotlib import pyplot as plt
all_sigmas_normalized = compute_normalized_deviations(
self.scaler.ISIGI, self.scaler.miller_set.indices())
plt.hist(all_sigmas_normalized, bins=100)
plt.figure()
binned_rms_normalized_sigmas = []
for i, sel in enumerate(sels):
binned_rms_normalized_sigmas.append(
math.sqrt(
flex.mean(
all_sigmas_normalized.select(sel) *
all_sigmas_normalized.select(sel))))
plt.plot(binned_intensities, binned_rms_normalized_sigmas, 'o')
plt.show()
all_sigmas_normalized = all_sigmas_normalized.select(
all_sigmas_normalized != 0)
self.normal_probability_plot(all_sigmas_normalized, (-0.5, 0.5),
plot=True)
def adjust_errors(self):
print("Starting adjust_errors", file=self.log)
# Save original sigmas
refls = self.scaler.ISIGI
refls['original_sigmas'] = refls['scaled_intensity'] / refls['isigi']
print("Computing initial estimates of parameters", file=self.log)
propagator = sdfac_propagate(self.scaler, verbose=False)
propagator.initial_estimates()
propagator.adjust_errors(compute_sums=False)
init_params = flex.double(self.get_initial_sdparams_estimates())
init_params.extend(propagator.error_terms.to_x())
values = self.parameterization(init_params)
print("Initial estimates:", end=' ', file=self.log)
values.show(self.log)
print("Refining error correction parameters", file=self.log)
sels, binned_intensities = self.get_binned_intensities()
minimizer = self.run_minimzer(values, sels)
values = minimizer.get_refined_params()
print("Final", end=' ', file=self.log)
values.show(self.log)
print("Applying sdfac/sdb/sdadd 1", file=self.log)
# Restore original sigmas
refls['isigi'] = refls['scaled_intensity'] / refls['original_sigmas']
# Propagate refined errors from postrefinement
propagator.error_terms = error_terms.from_x(values.propagate_terms)
propagator.adjust_errors()
minimizer.apply_sd_error_params(self.scaler.ISIGI, values)
self.scaler.summed_weight = flex.double(self.scaler.n_refl, 0.)
self.scaler.summed_wt_I = flex.double(self.scaler.n_refl, 0.)
print("Applying sdfac/sdb/sdadd 2", file=self.log)
for i in range(len(self.scaler.ISIGI)):
hkl_id = self.scaler.ISIGI['miller_id'][i]
Intensity = self.scaler.ISIGI['scaled_intensity'][
i] # scaled intensity
sigma = Intensity / self.scaler.ISIGI['isigi'][
i] # corrected sigma
variance = sigma * sigma
self.scaler.summed_wt_I[hkl_id] += Intensity / variance
self.scaler.summed_weight[hkl_id] += 1 / variance
if self.scaler.params.raw_data.error_models.sdfac_refine.plot_refinement_steps:
from matplotlib.pyplot import cm
from matplotlib import pyplot as plt
import numpy as np
for i in range(2):
f = plt.figure(i)
lines = plt.gca().get_lines()
color = cm.rainbow(np.linspace(0, 1, len(lines)))
for line, c in zip(reversed(lines), color):
line.set_color(c)
plt.ioff()
plt.show()
if False:
# validate using http://ccp4wiki.org/~ccp4wiki/wiki/index.php?title=Symmetry%2C_Scale%2C_Merge#Analysis_of_Standard_Deviations
print("Validating", file=self.log)
from matplotlib import pyplot as plt
all_sigmas_normalized = self.compute_normalized_deviations(
self.scaler.ISIGI, self.scaler.miller_set.indices())
plt.hist(all_sigmas_normalized, bins=100)
plt.figure()
binned_rms_normalized_sigmas = []
for i, sel in enumerate(sels):
binned_rms_normalized_sigmas.append(
math.sqrt(
flex.mean(
all_sigmas_normalized.select(sel) *
all_sigmas_normalized.select(sel))))
plt.plot(binned_intensities, binned_rms_normalized_sigmas, 'o')
plt.show()
all_sigmas_normalized = all_sigmas_normalized.select(
all_sigmas_normalized != 0)
self.normal_probability_plot(all_sigmas_normalized, (-0.5, 0.5),
plot=True)